[lgogdownloader] 01/05: New upstream version 3.0
Stephen Kitt
skitt at moszumanska.debian.org
Thu Nov 24 22:18:16 UTC 2016
This is an automated email from the git hooks/post-receive script.
skitt pushed a commit to branch master
in repository lgogdownloader.
commit f4312a5196c92ff8e5d36b6e67ce975ea1ccc4f3
Author: Stephen Kitt <steve at sk2.org>
Date: Wed Nov 23 23:10:26 2016 +0100
New upstream version 3.0
---
CMakeLists.txt | 34 +-
README.md | 30 +-
cmake/FindLibcrypto.cmake | 27 +
cmake/FindTinyxml.cmake | 33 -
cmake/FindTinyxml2.cmake | 33 +
include/blacklist.h | 2 +
include/config.h | 6 +
include/downloader.h | 24 +-
include/downloadinfo.h | 95 ++
include/globalconstants.h | 4 +-
include/message.h | 108 +++
include/ssl_thread_setup.h | 60 ++
include/threadsafequeue.h | 84 ++
include/util.h | 2 +
main.cpp | 70 +-
man/lgogdownloader.supplemental.groff | 11 +
src/downloader.cpp | 1664 ++++++++++++++++++++++-----------
src/gamedetails.cpp | 7 +
src/util.cpp | 69 +-
src/website.cpp | 20 +-
20 files changed, 1739 insertions(+), 644 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1670254..e94267c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,9 +1,12 @@
cmake_minimum_required(VERSION 3.0.0 FATAL_ERROR)
-project (lgogdownloader LANGUAGES CXX VERSION 2.28)
+project (lgogdownloader LANGUAGES C CXX VERSION 3.0)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG=1")
+set(LINK_LIBCRYPTO 0)
+find_program(READELF readelf DOC "Location of the readelf program")
+find_program(GREP grep DOC "Location of the grep program")
find_package(Boost
REQUIRED
system
@@ -13,11 +16,25 @@ find_package(Boost
date_time
)
find_package(CURL 7.32.0 REQUIRED)
+if(CURL_FOUND)
+ execute_process(
+ COMMAND ${READELF} -d ${CURL_LIBRARIES}
+ COMMAND ${GREP} -q "libssl\\|libcrypto"
+ RESULT_VARIABLE READELF_RESULT_VAR
+ )
+ if(READELF_RESULT_VAR EQUAL 0)
+ add_definitions(-DSSL_THREAD_SETUP_OPENSSL=1)
+ find_package(Libcrypto REQUIRED)
+ set(LINK_LIBCRYPTO 1)
+ endif(READELF_RESULT_VAR EQUAL 0)
+endif(CURL_FOUND)
+
find_package(OAuth REQUIRED)
find_package(Jsoncpp REQUIRED)
find_package(Htmlcxx REQUIRED)
-find_package(Tinyxml REQUIRED)
+find_package(Tinyxml2 REQUIRED)
find_package(Rhash REQUIRED)
+find_package(Threads REQUIRED)
file(GLOB SRC_FILES
main.cpp
@@ -83,7 +100,7 @@ target_include_directories(${PROJECT_NAME}
PRIVATE ${OAuth_INCLUDE_DIRS}
PRIVATE ${Jsoncpp_INCLUDE_DIRS}
PRIVATE ${Htmlcxx_INCLUDE_DIRS}
- PRIVATE ${Tinyxml_INCLUDE_DIRS}
+ PRIVATE ${Tinyxml2_INCLUDE_DIRS}
PRIVATE ${Rhash_INCLUDE_DIRS}
)
@@ -93,9 +110,16 @@ target_link_libraries(${PROJECT_NAME}
PRIVATE ${OAuth_LIBRARIES}
PRIVATE ${Jsoncpp_LIBRARIES}
PRIVATE ${Htmlcxx_LIBRARIES}
- PRIVATE ${Tinyxml_LIBRARIES}
+ PRIVATE ${Tinyxml2_LIBRARIES}
PRIVATE ${Rhash_LIBRARIES}
+ PRIVATE ${CMAKE_THREAD_LIBS_INIT}
+ )
+
+if(LINK_LIBCRYPTO EQUAL 1)
+ target_link_libraries(${PROJECT_NAME}
+ PRIVATE ${Libcrypto_LIBRARIES}
)
+endif(LINK_LIBCRYPTO EQUAL 1)
if(MSVC)
# Force to always compile with W4
@@ -106,7 +130,7 @@ if(MSVC)
endif()
elseif(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
# Update if necessary
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wno-long-long -fexceptions")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wextra -Wno-long-long -fexceptions")
endif()
set(INSTALL_BIN_DIR bin CACHE PATH "Installation directory for executables")
diff --git a/README.md b/README.md
index 0d28587..46b59ee 100644
--- a/README.md
+++ b/README.md
@@ -4,27 +4,33 @@ This repository contains the code of unofficial [GOG](http://www.gog.com/) downl
## Dependencies
-* libcurl
-* liboauth
-* librhash
-* jsoncpp
-* htmlcxx
-* tinyxml
-* boost (regex, date-time, system, filesystem, program-options)
-* help2man
+* [libcurl](https://curl.haxx.se/libcurl/) >= 7.32.0
+* [liboauth](https://sourceforge.net/projects/liboauth/)
+* [librhash](https://github.com/rhash/RHash)
+* [jsoncpp](https://github.com/open-source-parsers/jsoncpp)
+* [htmlcxx](http://htmlcxx.sourceforge.net/)
+* [tinyxml2](https://github.com/leethomason/tinyxml2)
+* [boost](http://www.boost.org/) (regex, date-time, system, filesystem, program-options)
+* [libcrypto](https://www.openssl.org/) if libcurl is built with OpenSSL
+
+## Make dependencies
+* [cmake](https://cmake.org/) >= 3.0.0
+* [help2man](https://www.gnu.org/software/help2man/help2man.html) (optional, man page generation)
+* [grep](https://www.gnu.org/software/grep/)
+* [binutils](https://www.gnu.org/software/binutils/) (readelf)
### Debian/Ubuntu
# apt install build-essential libcurl4-openssl-dev libboost-regex-dev \
- libjsoncpp-dev liboauth-dev librhash-dev libtinyxml-dev libhtmlcxx-dev\
- libboost-system-dev libboost-filesystem-dev libboost-program-options-dev\
- libboost-date-time-dev help2man cmake
+ libjsoncpp-dev liboauth-dev librhash-dev libtinyxml2-dev libhtmlcxx-dev \
+ libboost-system-dev libboost-filesystem-dev libboost-program-options-dev \
+ libboost-date-time-dev help2man cmake libssl-dev pkg-config
## Build and install
$ mkdir build
$ cd build
- $ cmake ..
+ $ cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release
$ make
# sudo make install
diff --git a/cmake/FindLibcrypto.cmake b/cmake/FindLibcrypto.cmake
new file mode 100644
index 0000000..3aa00b3
--- /dev/null
+++ b/cmake/FindLibcrypto.cmake
@@ -0,0 +1,27 @@
+# - Try to find libcrypto
+#
+# Once done this will define
+# Libcrypto_FOUND - System has libcrypto
+# Libcrypto_INCLUDE_DIRS - The libcrypto include directories
+# Libcrypto_LIBRARIES - The libraries needed to use libcrypto
+
+find_package(PkgConfig)
+pkg_check_modules(PC_LIBCRYPTO REQUIRED libcrypto)
+
+find_path(LIBCRYPTO_INCLUDE_DIR openssl/crypto.h
+ HINTS ${PC_LIBCRYPTO_INCLUDEDIR}
+ ${PC_LIBCRYPTO_INCLUDE_DIRS}
+ )
+
+find_library(LIBCRYPTO_LIBRARY NAMES crypto
+ HINTS ${PC_LIBCRYPTO_LIBDIR}
+ ${PC_LIBCRYPTO_LIBRARY_DIRS}
+ )
+
+mark_as_advanced(LIBCRYPTO_INCLUDE_DIR LIBCRYPTO_LIBRARY)
+
+if(PC_LIBCRYPTO_FOUND)
+ set(Libcrypto_FOUND ON)
+ set(Libcrypto_INCLUDE_DIRS ${LIBCRYPTO_INCLUDE_DIR})
+ set(Libcrypto_LIBRARIES ${LIBCRYPTO_LIBRARY})
+endif(PC_LIBCRYPTO_FOUND)
diff --git a/cmake/FindTinyxml.cmake b/cmake/FindTinyxml.cmake
deleted file mode 100644
index f399381..0000000
--- a/cmake/FindTinyxml.cmake
+++ /dev/null
@@ -1,33 +0,0 @@
-# - Try to find tinyxml
-#
-# Once done this will define
-# Tinyxml_FOUND - System has tinyxml
-# Tinyxml_INCLUDE_DIRS - The tinyxml include directories
-# Tinyxml_LIBRARIES - The libraries needed to use tinyxml
-
-find_package(PkgConfig)
-pkg_check_modules(PC_TINYXML tinyxml)
-
-find_path(TINYXML_INCLUDE_DIR tinyxml.h
- HINTS
- ${PC_TINYXML_INCLUDEDIR}
- ${PC_TINYXML_INCLUDE_DIRS}
- PATHS
- ${PC_TINYXML_INCLUDE_DIRS}
- )
-
-find_library(TINYXML_LIBRARY tinyxml
- HINTS
- ${PC_TINYXML_LIBDIR}
- ${PC_TINYXML_LIBRARY_DIRS}
- PATHS
- ${PC_TINYXML_LIBRARY_DIRS}
- )
-
-mark_as_advanced(TINYXML_INCLUDE_DIR TINYXML_LIBRARY)
-
-if(TINYXML_INCLUDE_DIR)
- set(Tinyxml_FOUND ON)
- set(Tinyxml_INCLUDE_DIRS ${TINYXML_INCLUDE_DIR})
- set(Tinyxml_LIBRARIES ${TINYXML_LIBRARY})
-endif(TINYXML_INCLUDE_DIR)
diff --git a/cmake/FindTinyxml2.cmake b/cmake/FindTinyxml2.cmake
new file mode 100644
index 0000000..4952977
--- /dev/null
+++ b/cmake/FindTinyxml2.cmake
@@ -0,0 +1,33 @@
+# - Try to find tinyxml2
+#
+# Once done this will define
+# Tinyxml2_FOUND - System has tinyxml2
+# Tinyxml2_INCLUDE_DIRS - The tinyxml2 include directories
+# Tinyxml2_LIBRARIES - The libraries needed to use tinyxml
+
+find_package(PkgConfig)
+pkg_check_modules(PC_TINYXML2 tinyxml2)
+
+find_path(TINYXML2_INCLUDE_DIR tinyxml2.h
+ HINTS
+ ${PC_TINYXML2_INCLUDEDIR}
+ ${PC_TINYXML2_INCLUDE_DIRS}
+ PATHS
+ ${PC_TINYXML2_INCLUDE_DIRS}
+ )
+
+find_library(TINYXML2_LIBRARY tinyxml2
+ HINTS
+ ${PC_TINYXML2_LIBDIR}
+ ${PC_TINYXML2_LIBRARY_DIRS}
+ PATHS
+ ${PC_TINYXML2_LIBRARY_DIRS}
+ )
+
+mark_as_advanced(TINYXML2_INCLUDE_DIR TINYXML2_LIBRARY)
+
+if(TINYXML2_INCLUDE_DIR)
+ set(Tinyxml2_FOUND ON)
+ set(Tinyxml2_INCLUDE_DIRS ${TINYXML2_INCLUDE_DIR})
+ set(Tinyxml2_LIBRARIES ${TINYXML2_LIBRARY})
+endif(TINYXML2_INCLUDE_DIR)
diff --git a/include/blacklist.h b/include/blacklist.h
index 0799948..9cb49b0 100644
--- a/include/blacklist.h
+++ b/include/blacklist.h
@@ -31,6 +31,8 @@ class Blacklist
bool isBlacklisted(const std::string& path);
bool isBlacklisted(const std::string& path, const std::string& gamename, std::string subdirectory = "");
+ std::vector<BlacklistItem>::size_type size() const { return blacklist_.size(); }
+ bool empty() { return blacklist_.empty(); }
private:
std::vector<BlacklistItem> blacklist_;
};
diff --git a/include/config.h b/include/config.h
index 75cc478..2881f3e 100644
--- a/include/config.h
+++ b/include/config.h
@@ -48,6 +48,7 @@ class Config
bool bShowWishlist;
bool bAutomaticXMLCreation;
bool bSaveChangelogs;
+ bool bRespectUmask;
std::string sGameRegex;
std::string sDirectory;
std::string sCacheDirectory;
@@ -62,8 +63,10 @@ class Config
std::string sConfigFilePath;
std::string sBlacklistFilePath;
std::string sIgnorelistFilePath;
+ std::string sGameHasDLCListFilePath;
std::string sOrphanRegex;
std::string sCoverList;
+ std::string sGameHasDLCList;
std::string sReportFilePath;
std::string sInstallersSubdir;
std::string sExtrasSubdir;
@@ -76,12 +79,14 @@ class Config
std::string sLanguagePriority;
std::string sPlatformPriority;
std::string sIgnoreDLCCountRegex;
+ std::string sCACertPath;
std::vector<unsigned int> vLanguagePriority;
std::vector<unsigned int> vPlatformPriority;
unsigned int iInstallerPlatform;
unsigned int iInstallerLanguage;
unsigned int iInclude;
+ unsigned int iThreads;
int iRetries;
int iWait;
int iCacheValid;
@@ -90,6 +95,7 @@ class Config
long int iTimeout;
Blacklist blacklist;
Blacklist ignorelist;
+ Blacklist gamehasdlc;
};
#endif // CONFIG_H__
diff --git a/include/downloader.h b/include/downloader.h
index a85a1f8..0aa1d45 100644
--- a/include/downloader.h
+++ b/include/downloader.h
@@ -25,6 +25,7 @@
#include "api.h"
#include "progressbar.h"
#include "website.h"
+#include "threadsafequeue.h"
#include <curl/curl.h>
#include <json/json.h>
#include <ctime>
@@ -48,6 +49,15 @@ class Timer
struct timeval last_update;
};
+struct xferInfo
+{
+ unsigned int tid;
+ CURL* curlhandle;
+ Timer timer;
+ std::deque< std::pair<time_t, uintmax_t> > TimeAndSize;
+ curl_off_t offset;
+};
+
class Downloader
{
public:
@@ -55,14 +65,14 @@ class Downloader
virtual ~Downloader();
int init();
int login();
- void listGames();
+ int listGames();
void updateCheck();
void repair();
void download();
void checkOrphans();
void checkStatus();
void updateCache();
- void downloadFileWithId(const std::string& fileid_string, const std::string& output_filepath);
+ int downloadFileWithId(const std::string& fileid_string, const std::string& output_filepath);
void showWishlist();
CURL* curlhandle;
Timer timer;
@@ -84,11 +94,15 @@ class Downloader
int loadGameDetailsCache();
int saveGameDetailsCache();
std::vector<gameDetails> getGameDetailsFromJsonNode(Json::Value root, const int& recursion_level = 0);
- std::vector<gameFile> getExtrasFromJSON(const Json::Value& json, const std::string& gamename);
- std::string getSerialsFromJSON(const Json::Value& json);
+ static std::vector<gameFile> getExtrasFromJSON(const Json::Value& json, const std::string& gamename, const Config& config);
+ static std::string getSerialsFromJSON(const Json::Value& json);
void saveSerials(const std::string& serials, const std::string& filepath);
- std::string getChangelogFromJSON(const Json::Value& json);
+ static std::string getChangelogFromJSON(const Json::Value& json);
void saveChangelog(const std::string& changelog, const std::string& filepath);
+ static void processDownloadQueue(Config conf, const unsigned int& tid);
+ static int progressCallbackForThread(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow);
+ void printProgress();
+ static void getGameDetailsThread(Config config, const unsigned int& tid);
static int progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow);
static size_t writeMemoryCallback(char *ptr, size_t size, size_t nmemb, void *userp);
diff --git a/include/downloadinfo.h b/include/downloadinfo.h
new file mode 100644
index 0000000..6015f89
--- /dev/null
+++ b/include/downloadinfo.h
@@ -0,0 +1,95 @@
+/* This program is free software. It comes without any warranty, to
+ * the extent permitted by applicable law. You can redistribute it
+ * and/or modify it under the terms of the Do What The Fuck You Want
+ * To Public License, Version 2, as published by Sam Hocevar. See
+ * http://www.wtfpl.net/ for more details. */
+
+#ifndef DOWNLOADINFO_H
+#define DOWNLOADINFO_H
+
+#include <curl/curl.h>
+#include <mutex>
+
+const unsigned int DLSTATUS_NOTSTARTED = 0;
+const unsigned int DLSTATUS_STARTING = 1 << 0;
+const unsigned int DLSTATUS_RUNNING = 1 << 1;
+const unsigned int DLSTATUS_FINISHED = 1 << 2;
+
+struct progressInfo
+{
+ curl_off_t dlnow;
+ curl_off_t dltotal;
+ double rate;
+ double rate_avg;
+};
+
+class DownloadInfo
+{
+ public:
+ void setFilename(const std::string& filename_)
+ {
+ std::unique_lock<std::mutex> lock(m);
+ filename = filename_;
+ }
+
+ std::string getFilename()
+ {
+ std::unique_lock<std::mutex> lock(m);
+ return filename;
+ }
+
+ void setStatus(const unsigned int& status_)
+ {
+ std::unique_lock<std::mutex> lock(m);
+ status = status_;
+ }
+
+ unsigned int getStatus()
+ {
+ std::unique_lock<std::mutex> lock(m);
+ return status;
+ }
+
+ void setProgressInfo(const progressInfo& info)
+ {
+ std::unique_lock<std::mutex> lock(m);
+ progress_info = info;
+ }
+
+ progressInfo getProgressInfo()
+ {
+ std::unique_lock<std::mutex> lock(m);
+ return progress_info;
+ }
+
+ DownloadInfo()=default;
+
+ DownloadInfo(const DownloadInfo& other)
+ {
+ std::lock_guard<std::mutex> guard(other.m);
+ filename = other.filename;
+ status = other.status;
+ progress_info = other.progress_info;
+ }
+
+ DownloadInfo& operator= (DownloadInfo& other)
+ {
+ if(&other == this)
+ return *this;
+
+ std::unique_lock<std::mutex> lock1(m, std::defer_lock);
+ std::unique_lock<std::mutex> lock2(other.m, std::defer_lock);
+ std::lock(lock1, lock2);
+ filename = other.filename;
+ status = other.status;
+ progress_info = other.progress_info;
+ return *this;
+ }
+ private:
+ std::string filename;
+ unsigned int status;
+ progressInfo progress_info;
+ mutable std::mutex m;
+};
+
+#endif // DOWNLOADINFO_H
diff --git a/include/globalconstants.h b/include/globalconstants.h
index 935ac2f..52b9dc1 100644
--- a/include/globalconstants.h
+++ b/include/globalconstants.h
@@ -39,6 +39,7 @@ namespace GlobalConstants
const unsigned int LANGUAGE_FI = 1 << 18;
const unsigned int LANGUAGE_PT_BR = 1 << 19;
const unsigned int LANGUAGE_SK = 1 << 20;
+ const unsigned int LANGUAGE_BL = 1 << 21;
const std::vector<optionsStruct> LANGUAGES =
{
@@ -62,7 +63,8 @@ namespace GlobalConstants
{ LANGUAGE_DA, "da", "Danish" , "da|dan|danish" },
{ LANGUAGE_FI, "fi", "Finnish" , "fi|fin|finnish" },
{ LANGUAGE_PT_BR, "br", "Brazilian Portuguese", "br|pt_br|pt-br|ptbr|brazilian_portuguese" },
- { LANGUAGE_SK, "sk", "Slovak" , "sk|slk|slo|slovak" }
+ { LANGUAGE_SK, "sk", "Slovak" , "sk|slk|slo|slovak" },
+ { LANGUAGE_BL, "bl", "Bulgarian" , "bl|bg|bul|bulgarian" }
};
// Platform constants
diff --git a/include/message.h b/include/message.h
new file mode 100644
index 0000000..1704ccb
--- /dev/null
+++ b/include/message.h
@@ -0,0 +1,108 @@
+/* This program is free software. It comes without any warranty, to
+ * the extent permitted by applicable law. You can redistribute it
+ * and/or modify it under the terms of the Do What The Fuck You Want
+ * To Public License, Version 2, as published by Sam Hocevar. See
+ * http://www.wtfpl.net/ for more details. */
+
+#ifndef MESSAGE_H
+#define MESSAGE_H
+
+#include <boost/date_time/posix_time/posix_time.hpp>
+
+const unsigned int MSGTYPE_INFO = 1 << 0;
+const unsigned int MSGTYPE_WARNING = 1 << 1;
+const unsigned int MSGTYPE_ERROR = 1 << 2;
+const unsigned int MSGTYPE_SUCCESS = 1 << 3;
+
+class Message
+{
+ public:
+ Message() = default;
+ Message(std::string msg, const unsigned int& type = MSGTYPE_INFO, const std::string& prefix = std::string())
+ {
+ prefix_ = prefix;
+ msg_ = msg;
+ type_ = type;
+ timestamp_ = boost::posix_time::second_clock::local_time();
+ }
+
+ void setMessage(const std::string& msg)
+ {
+ msg_ = msg;
+ }
+
+ void setType(const unsigned int& type)
+ {
+ type_ = type;
+ }
+
+ void setTimestamp(const boost::posix_time::ptime& timestamp)
+ {
+ timestamp_ = timestamp;
+ }
+
+ void setPrefix(const std::string& prefix)
+ {
+ prefix_ = prefix;
+ }
+
+ std::string getMessage()
+ {
+ return msg_;
+ }
+
+ unsigned int getType()
+ {
+ return type_;
+ }
+
+ boost::posix_time::ptime getTimestamp()
+ {
+ return timestamp_;
+ }
+
+ std::string getTimestampString()
+ {
+ return boost::posix_time::to_simple_string(timestamp_);
+ }
+
+ std::string getPrefix()
+ {
+ return prefix_;
+ }
+
+ std::string getFormattedString(const bool& bColor = true, const bool& bPrefix = true)
+ {
+ std::string str;
+ std::string color_value = "\033[39m"; // Default foreground color
+ std::string color_reset = "\033[0m";
+
+ if (type_ == MSGTYPE_INFO)
+ color_value = "\033[39m"; // Default foreground color
+ else if (type_ == MSGTYPE_WARNING)
+ color_value = "\033[33m"; // Yellow
+ else if (type_ == MSGTYPE_ERROR)
+ color_value = "\033[31m"; // Red
+ else if (type_ == MSGTYPE_SUCCESS)
+ color_value = "\033[32m"; // Green
+
+ str = msg_;
+ if (!prefix_.empty() && bPrefix)
+ str = prefix_ + " " + str;
+
+ str = getTimestampString() + " " + str;
+
+ if (bColor)
+ str = color_value + str + color_reset;
+
+ return str;
+ }
+
+ private:
+ std::string msg_;
+ boost::posix_time::ptime timestamp_;
+ unsigned int type_;
+ std::string prefix_;
+};
+
+#endif // MESSAGE_H
diff --git a/include/ssl_thread_setup.h b/include/ssl_thread_setup.h
new file mode 100644
index 0000000..6a5aee4
--- /dev/null
+++ b/include/ssl_thread_setup.h
@@ -0,0 +1,60 @@
+/* This program is free software. It comes without any warranty, to
+ * the extent permitted by applicable law. You can redistribute it
+ * and/or modify it under the terms of the Do What The Fuck You Want
+ * To Public License, Version 2, as published by Sam Hocevar. See
+ * http://www.wtfpl.net/ for more details. */
+
+#ifndef SSL_THREAD_SETUP_H
+#define SSL_THREAD_SETUP_H
+
+#include <thread>
+#include <mutex>
+
+#if SSL_THREAD_SETUP_OPENSSL == 1
+ #include <openssl/crypto.h>
+
+ static std::mutex* ssl_mutex_array;
+
+ void thread_locking_callback(int mode, int n, const char* file, int line)
+ {
+ if(mode & CRYPTO_LOCK)
+ ssl_mutex_array[n].lock();
+ else
+ ssl_mutex_array[n].unlock();
+ }
+
+ unsigned long thread_id_callback()
+ {
+ return (unsigned long)std::hash<std::thread::id>() (std::this_thread::get_id());
+ }
+
+ int ssl_thread_setup()
+ {
+ ssl_mutex_array = new std::mutex[CRYPTO_num_locks()];
+ if(!ssl_mutex_array)
+ return 0;
+ else
+ {
+ CRYPTO_set_id_callback(thread_id_callback);
+ CRYPTO_set_locking_callback(thread_locking_callback);
+ }
+ return 1;
+ }
+
+ int ssl_thread_cleanup()
+ {
+ if(!ssl_mutex_array)
+ return 0;
+
+ CRYPTO_set_id_callback(NULL);
+ CRYPTO_set_locking_callback(NULL);
+ delete[] ssl_mutex_array;
+ ssl_mutex_array = NULL;
+ return 1;
+ }
+#else
+ #define ssl_thread_setup()
+ #define ssl_thread_cleanup()
+#endif
+
+#endif // SSL_THREAD_SETUP_H
diff --git a/include/threadsafequeue.h b/include/threadsafequeue.h
new file mode 100644
index 0000000..a456daa
--- /dev/null
+++ b/include/threadsafequeue.h
@@ -0,0 +1,84 @@
+/* This program is free software. It comes without any warranty, to
+ * the extent permitted by applicable law. You can redistribute it
+ * and/or modify it under the terms of the Do What The Fuck You Want
+ * To Public License, Version 2, as published by Sam Hocevar. See
+ * http://www.wtfpl.net/ for more details. */
+
+#ifndef THREADSAFEQUEUE_H
+#define THREADSAFEQUEUE_H
+
+#include <queue>
+#include <mutex>
+#include <condition_variable>
+
+template<typename T>
+class ThreadSafeQueue
+{
+ public:
+ void push(const T& item)
+ {
+ std::unique_lock<std::mutex> lock(m);
+ q.push(item);
+ lock.unlock();
+ cvar.notify_one();
+ }
+
+ bool empty() const
+ {
+ std::unique_lock<std::mutex> lock(m);
+ return q.empty();
+ }
+
+ typename std::queue<T>::size_type size() const
+ {
+ std::unique_lock<std::mutex> lock(m);
+ return q.size();
+ }
+
+ bool try_pop(T& item)
+ {
+ std::unique_lock<std::mutex> lock(m);
+ if(q.empty())
+ return false;
+
+ item = q.front();
+ q.pop();
+ return true;
+ }
+
+ void wait_and_pop(T& item)
+ {
+ std::unique_lock<std::mutex> lock(m);
+ while(q.empty())
+ cvar.wait(lock);
+
+ item = q.front();
+ q.pop();
+ }
+
+ ThreadSafeQueue() = default;
+
+ ThreadSafeQueue(const ThreadSafeQueue& other)
+ {
+ std::lock_guard<std::mutex> guard(other.m);
+ q = other.q;
+ }
+
+ ThreadSafeQueue& operator= (ThreadSafeQueue& other)
+ {
+ if(&other == this)
+ return *this;
+
+ std::unique_lock<std::mutex> lock1(m, std::defer_lock);
+ std::unique_lock<std::mutex> lock2(other.m, std::defer_lock);
+ std::lock(lock1, lock2);
+ q = other.q;
+ return *this;
+ }
+ private:
+ std::queue<T> q;
+ mutable std::mutex m;
+ std::condition_variable cvar;
+};
+
+#endif // THREADSAFEQUEUE_H
diff --git a/include/util.h b/include/util.h
index 7135a5a..f8e4176 100644
--- a/include/util.h
+++ b/include/util.h
@@ -88,6 +88,8 @@ namespace Util
unsigned int getOptionValue(const std::string& str, const std::vector<GlobalConstants::optionsStruct>& options);
std::string getOptionNameString(const unsigned int& value, const std::vector<GlobalConstants::optionsStruct>& options);
void parseOptionString(const std::string &option_string, std::vector<unsigned int> &priority, unsigned int &type, const std::vector<GlobalConstants::optionsStruct>& options);
+ std::string getLocalFileHash(const std::string& xml_dir, const std::string& filepath, const std::string& gamename = std::string());
+ void shortenStringToTerminalWidth(std::string& str);
}
#endif // UTIL_H
diff --git a/main.cpp b/main.cpp
index 3307dcc..5a761fd 100644
--- a/main.cpp
+++ b/main.cpp
@@ -55,6 +55,7 @@ int main(int argc, char *argv[])
config.sConfigFilePath = config.sConfigDirectory + "/config.cfg";
config.sBlacklistFilePath = config.sConfigDirectory + "/blacklist.txt";
config.sIgnorelistFilePath = config.sConfigDirectory + "/ignorelist.txt";
+ config.sGameHasDLCListFilePath = config.sConfigDirectory + "/game_has_dlc.txt";
std::string priority_help_text = "Set priority by separating values with \",\"\nCombine values by separating with \"+\"";
// Create help text for --platform option
@@ -143,6 +144,8 @@ int main(int argc, char *argv[])
("wishlist", bpo::value<bool>(&config.bShowWishlist)->zero_tokens()->default_value(false), "Show wishlist")
("login-api", bpo::value<bool>(&config.bLoginAPI)->zero_tokens()->default_value(false), "Login (API only)")
("login-website", bpo::value<bool>(&config.bLoginHTTP)->zero_tokens()->default_value(false), "Login (website only)")
+ ("cacert", bpo::value<std::string>(&config.sCACertPath)->default_value(""), "Path to CA certificate bundle in PEM format")
+ ("respect-umask", bpo::value<bool>(&config.bRespectUmask)->zero_tokens()->default_value(false), "Do not adjust permissions of sensitive files")
;
// Commandline options (config file)
options_cli_cfg.add_options()
@@ -154,7 +157,7 @@ int main(int argc, char *argv[])
("language", bpo::value<std::string>(&sInstallerLanguage)->default_value("en"), language_text.c_str())
("no-remote-xml", bpo::value<bool>(&bNoRemoteXML)->zero_tokens()->default_value(false), "Don't use remote XML for repair")
("no-unicode", bpo::value<bool>(&bNoUnicode)->zero_tokens()->default_value(false), "Don't use Unicode in the progress bar")
- ("no-color", bpo::value<bool>(&bNoColor)->zero_tokens()->default_value(false), "Don't use coloring in the progress bar")
+ ("no-color", bpo::value<bool>(&bNoColor)->zero_tokens()->default_value(false), "Don't use coloring in the progress bar or status messages")
("no-duplicate-handling", bpo::value<bool>(&bNoDuplicateHandler)->zero_tokens()->default_value(false), "Don't use duplicate handler for installers\nDuplicate installers from different languages are handled separately")
("no-subdirectories", bpo::value<bool>(&bNoSubDirectories)->zero_tokens()->default_value(false), "Don't create subdirectories for extras, patches and language packs")
("verbose", bpo::value<bool>(&config.bVerbose)->zero_tokens()->default_value(false), "Print lots of information")
@@ -162,7 +165,7 @@ int main(int argc, char *argv[])
("timeout", bpo::value<long int>(&config.iTimeout)->default_value(10), "Set timeout for connection\nMaximum time in seconds that connection phase is allowed to take")
("retries", bpo::value<int>(&config.iRetries)->default_value(3), "Set maximum number of retries on failed download")
("wait", bpo::value<int>(&config.iWait)->default_value(0), "Time to wait between requests (milliseconds)")
- ("cover-list", bpo::value<std::string>(&config.sCoverList)->default_value("https://sites.google.com/site/gogdownloader/covers.xml"), "Set URL for cover list")
+ ("cover-list", bpo::value<std::string>(&config.sCoverList)->default_value("https://raw.githubusercontent.com/Sude-/lgogdownloader-lists/master/covers.xml"), "Set URL for cover list")
("subdir-installers", bpo::value<std::string>(&config.sInstallersSubdir)->default_value(""), ("Set subdirectory for extras" + subdir_help_text).c_str())
("subdir-extras", bpo::value<std::string>(&config.sExtrasSubdir)->default_value("extras"), ("Set subdirectory for extras" + subdir_help_text).c_str())
("subdir-patches", bpo::value<std::string>(&config.sPatchesSubdir)->default_value("patches"), ("Set subdirectory for patches" + subdir_help_text).c_str())
@@ -177,6 +180,8 @@ int main(int argc, char *argv[])
("exclude", bpo::value<std::string>(&sExcludeOptions)->default_value("covers"), ("Select what not to download/list/repair\n" + include_options_text).c_str())
("automatic-xml-creation", bpo::value<bool>(&config.bAutomaticXMLCreation)->zero_tokens()->default_value(false), "Automatically create XML data after download has completed")
("save-changelogs", bpo::value<bool>(&config.bSaveChangelogs)->zero_tokens()->default_value(false), "Save changelogs when downloading")
+ ("threads", bpo::value<unsigned int>(&config.iThreads)->default_value(4), "Number of download threads")
+ ("dlc-list", bpo::value<std::string>(&config.sGameHasDLCList)->default_value("https://raw.githubusercontent.com/Sude-/lgogdownloader-lists/master/game_has_dlc.txt"), "Set URL for list of games that have DLC")
;
// Options read from config file
options_cfg_only.add_options()
@@ -295,6 +300,30 @@ int main(int argc, char *argv[])
}
}
+ if (config.sIgnoreDLCCountRegex.empty())
+ {
+ if (boost::filesystem::exists(config.sGameHasDLCListFilePath))
+ {
+ std::ifstream ifs(config.sGameHasDLCListFilePath.c_str());
+ if (!ifs)
+ {
+ std::cerr << "Could not open list of games that have dlc: " << config.sGameHasDLCListFilePath << std::endl;
+ return 1;
+ }
+ else
+ {
+ std::string line;
+ std::vector<std::string> lines;
+ while (!ifs.eof())
+ {
+ std::getline(ifs, line);
+ lines.push_back(std::move(line));
+ }
+ config.gamehasdlc.initialize(lines);
+ }
+ }
+ }
+
if (vm.count("chunk-size"))
config.iChunkSize <<= 20; // Convert chunk size from bytes to megabytes
@@ -311,6 +340,12 @@ int main(int argc, char *argv[])
if (config.iWait > 0)
config.iWait *= 1000;
+ if (config.iThreads < 1)
+ {
+ config.iThreads = 1;
+ set_vm_value(vm, "threads", config.iThreads);
+ }
+
config.bVerifyPeer = !bInsecure;
config.bColor = !bNoColor;
config.bUnicode = !bNoUnicode;
@@ -421,6 +456,15 @@ int main(int argc, char *argv[])
config.sDirectory = "./"; // Directory wasn't specified, use current directory
}
+ // CA certificate bundle
+ if (config.sCACertPath.empty())
+ {
+ // Use CURL_CA_BUNDLE environment variable for CA certificate path if it is set
+ char *ca_bundle = getenv("CURL_CA_BUNDLE");
+ if (ca_bundle)
+ config.sCACertPath = (std::string)ca_bundle;
+ }
+
if (!unrecognized_options_cfg.empty() && (!config.bSaveConfig || !config.bResetConfig))
{
std::cerr << "Unrecognized options in " << config.sConfigFilePath << std::endl;
@@ -445,8 +489,11 @@ int main(int argc, char *argv[])
}
// Make sure that config file and cookie file are only readable/writable by owner
- Util::setFilePermissions(config.sConfigFilePath, boost::filesystem::owner_read | boost::filesystem::owner_write);
- Util::setFilePermissions(config.sCookiePath, boost::filesystem::owner_read | boost::filesystem::owner_write);
+ if (!config.bRespectUmask)
+ {
+ Util::setFilePermissions(config.sConfigFilePath, boost::filesystem::owner_read | boost::filesystem::owner_write);
+ Util::setFilePermissions(config.sCookiePath, boost::filesystem::owner_read | boost::filesystem::owner_write);
+ }
if (config.bSaveConfig || iLoginResult == 1)
{
@@ -503,7 +550,8 @@ int main(int argc, char *argv[])
}
}
ofs.close();
- Util::setFilePermissions(config.sConfigFilePath, boost::filesystem::owner_read | boost::filesystem::owner_write);
+ if (!config.bRespectUmask)
+ Util::setFilePermissions(config.sConfigFilePath, boost::filesystem::owner_read | boost::filesystem::owner_write);
if (config.bSaveConfig)
return 0;
}
@@ -524,7 +572,8 @@ int main(int argc, char *argv[])
ofs << "secret = " << config.sSecret << std::endl;
}
ofs.close();
- Util::setFilePermissions(config.sConfigFilePath, boost::filesystem::owner_read | boost::filesystem::owner_write);
+ if (!config.bRespectUmask)
+ Util::setFilePermissions(config.sConfigFilePath, boost::filesystem::owner_read | boost::filesystem::owner_write);
return 0;
}
else
@@ -533,6 +582,9 @@ int main(int argc, char *argv[])
return 1;
}
}
+
+ int res = 0;
+
if (config.bShowWishlist)
downloader.showWishlist();
else if (config.bUpdateCache)
@@ -543,7 +595,7 @@ int main(int argc, char *argv[])
{
for (std::vector<std::string>::iterator it = vFileIdStrings.begin(); it != vFileIdStrings.end(); it++)
{
- downloader.downloadFileWithId(*it, config.sOutputFilename);
+ res |= downloader.downloadFileWithId(*it, config.sOutputFilename) ? 1 : 0;
}
}
else if (config.bRepair) // Repair file
@@ -551,7 +603,7 @@ int main(int argc, char *argv[])
else if (config.bDownload) // Download games
downloader.download();
else if (config.bListDetails || config.bList) // Detailed list of games/extras
- downloader.listGames();
+ res = downloader.listGames();
else if (!config.sOrphanRegex.empty()) // Check for orphaned files if regex for orphans is set
downloader.checkOrphans();
else if (config.bCheckStatus)
@@ -570,5 +622,5 @@ int main(int argc, char *argv[])
if (!config.sOrphanRegex.empty() && config.bDownload)
downloader.checkOrphans();
- return 0;
+ return res;
}
diff --git a/man/lgogdownloader.supplemental.groff b/man/lgogdownloader.supplemental.groff
index a376412..d3d4800 100644
--- a/man/lgogdownloader.supplemental.groff
+++ b/man/lgogdownloader.supplemental.groff
@@ -75,6 +75,17 @@ blacklist.
It doesn't have to exist, but if it does exist, it must be readable to lgogdownloader.
.TP
+\fI$XDG_CONFIG_HOME/lgogdownloader/game_has_dlc.txt\fP
+Allows user to specify which games have dlc and should have their DLC count
+information ignored. The file has the same format and interpretation as a
+blacklist.
+.br
+It doesn't have to exist, but if it does exist, it must be readable to lgogdownloader.
+.br
+If the file exists lgogdownloader uses it instead of list specified with
+\fB--dlc-list\fP option
+
+.TP
\fI$XDG_CONFIG_HOME/lgogdownloader/gamespecific/gamename.conf\fP
JSON formatted file. Sets game specific settings for \fBgamename\fP.
.br
diff --git a/src/downloader.cpp b/src/downloader.cpp
index ccb43d1..9674cc2 100644
--- a/src/downloader.cpp
+++ b/src/downloader.cpp
@@ -7,6 +7,9 @@
#include "downloader.h"
#include "util.h"
#include "globalconstants.h"
+#include "ssl_thread_setup.h"
+#include "downloadinfo.h"
+#include "message.h"
#include <cstdio>
#include <cstdlib>
@@ -19,15 +22,26 @@
#include <boost/filesystem.hpp>
#include <boost/regex.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
-#include <tinyxml.h>
+#include <tinyxml2.h>
#include <json/json.h>
#include <htmlcxx/html/ParserDom.h>
#include <htmlcxx/html/Uri.h>
+#include <termios.h>
+#include <algorithm>
namespace bptime = boost::posix_time;
+std::vector<DownloadInfo> vDownloadInfo;
+ThreadSafeQueue<gameFile> dlQueue;
+ThreadSafeQueue<Message> msgQueue;
+ThreadSafeQueue<gameFile> createXMLQueue;
+ThreadSafeQueue<gameItem> gameItemQueue;
+ThreadSafeQueue<gameDetails> gameDetailsQueue;
+std::mutex mtx_create_directories; // Mutex for creating directories in Downloader::processDownloadQueue
+
Downloader::Downloader(Config &conf)
{
+ ssl_thread_setup();
this->config = conf;
if (config.bLoginHTTP && boost::filesystem::exists(config.sCookiePath))
if (!boost::filesystem::remove(config.sCookiePath))
@@ -44,8 +58,10 @@ Downloader::~Downloader()
delete gogWebsite;
curl_easy_cleanup(curlhandle);
curl_global_cleanup();
+ ssl_thread_cleanup();
// Make sure that cookie file is only readable/writable by owner
- Util::setFilePermissions(config.sCookiePath, boost::filesystem::owner_read | boost::filesystem::owner_write);
+ if (!config.bRespectUmask)
+ Util::setFilePermissions(config.sCookiePath, boost::filesystem::owner_read | boost::filesystem::owner_write);
}
@@ -78,6 +94,9 @@ int Downloader::init()
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_TIME, 30);
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_LIMIT, 200);
+ if (!config.sCACertPath.empty())
+ curl_easy_setopt(curlhandle, CURLOPT_CAINFO, config.sCACertPath.c_str());
+
// Create new GOG website handle
gogWebsite = new Website(config);
bool bWebsiteIsLoggedIn = gogWebsite->IsLoggedIn();
@@ -87,6 +106,8 @@ int Downloader::init()
gogAPI->curlSetOpt(CURLOPT_VERBOSE, config.bVerbose);
gogAPI->curlSetOpt(CURLOPT_SSL_VERIFYPEER, config.bVerifyPeer);
gogAPI->curlSetOpt(CURLOPT_CONNECTTIMEOUT, config.iTimeout);
+ if (!config.sCACertPath.empty())
+ gogAPI->curlSetOpt(CURLOPT_CAINFO, config.sCACertPath.c_str());
progressbar = new ProgressBar(config.bUnicode, config.bColor);
@@ -94,6 +115,17 @@ int Downloader::init()
if (!bInitOK || !bWebsiteIsLoggedIn || config.bLoginHTTP || config.bLoginAPI)
return 1;
+ if (!config.sGameHasDLCList.empty())
+ {
+ if (config.gamehasdlc.empty())
+ {
+ std::string game_has_dlc_list = this->getResponse(config.sGameHasDLCList);
+ if (!game_has_dlc_list.empty())
+ config.gamehasdlc.initialize(Util::tokenize(game_has_dlc_list, "\n"));
+ }
+ }
+ gogWebsite->setConfig(config); // Update config for website handle
+
if (config.bCover && config.bDownload && !config.bUpdateCheck)
coverXML = this->getResponse(config.sCoverList);
@@ -121,7 +153,6 @@ int Downloader::init()
*/
int Downloader::login()
{
- char *pwd;
std::string email;
if (!isatty(STDIN_FILENO)) {
std::cerr << "Unable to read email and password" << std::endl;
@@ -129,8 +160,18 @@ int Downloader::login()
}
std::cerr << "Email: ";
std::getline(std::cin,email);
- pwd = getpass("Password: ");
- std::string password = (std::string)pwd;
+
+ std::string password;
+ std::cerr << "Password: ";
+ struct termios termios_old, termios_new;
+ tcgetattr(STDIN_FILENO, &termios_old); // Get current terminal attributes
+ termios_new = termios_old;
+ termios_new.c_lflag &= ~ECHO; // Set ECHO off
+ tcsetattr(STDIN_FILENO, TCSANOW, &termios_new); // Set terminal attributes
+ std::getline(std::cin, password);
+ tcsetattr(STDIN_FILENO, TCSANOW, &termios_old); // Restore old terminal attributes
+ std::cerr << std::endl;
+
if (email.empty() || password.empty())
{
std::cerr << "Email and/or password empty" << std::endl;
@@ -182,6 +223,7 @@ void Downloader::updateCheck()
if (gogAPI->user.notifications_games)
{
config.sGameRegex = ".*"; // Always check all games
+ gogWebsite->setConfig(config); // Make sure that website handle has updated config
if (config.bList || config.bListDetails || config.bDownload)
{
if (config.bList)
@@ -265,217 +307,89 @@ int Downloader::getGameDetails()
}
}
- gameDetails game;
- int updated = 0;
- for (unsigned int i = 0; i < gameItems.size(); ++i)
+ if (!gameItems.empty())
{
- std::cerr << "Getting game info " << i+1 << " / " << gameItems.size() << "\r" << std::flush;
- bool bHasDLC = !gameItems[i].dlcnames.empty();
+ for (unsigned int i = 0; i < gameItems.size(); ++i)
+ {
+ gameItemQueue.push(gameItems[i]);
+ }
- gameSpecificConfig conf;
- conf.bDLC = config.bDLC;
- conf.bIgnoreDLCCount = false;
- conf.iInstallerLanguage = config.iInstallerLanguage;
- conf.iInstallerPlatform = config.iInstallerPlatform;
- conf.dirConf = dirConfDefault;
- conf.vLanguagePriority = config.vLanguagePriority;
- conf.vPlatformPriority = config.vPlatformPriority;
- if (!config.bUpdateCache) // Disable game specific config files for cache update
+ // Create threads
+ unsigned int threads = config.iThreads;
+ if (gameItemQueue.size() < config.iThreads)
{
- int iOptionsOverridden = Util::getGameSpecificConfig(gameItems[i].name, &conf);
- if (iOptionsOverridden > 0)
- {
- std::cerr << std::endl << gameItems[i].name << " - " << iOptionsOverridden << " options overridden with game specific options" << std::endl;
- if (config.bVerbose)
- {
- if (conf.bIgnoreDLCCount)
- std::cerr << "\tIgnore DLC count" << std::endl;
- if (conf.bDLC != config.bDLC)
- std::cerr << "\tDLC: " << (conf.bDLC ? "true" : "false") << std::endl;
- if (conf.iInstallerLanguage != config.iInstallerLanguage)
- std::cerr << "\tLanguage: " << Util::getOptionNameString(conf.iInstallerLanguage, GlobalConstants::LANGUAGES) << std::endl;
- if (conf.vLanguagePriority != config.vLanguagePriority)
- {
- std::cerr << "\tLanguage priority:" << std::endl;
- for (unsigned int j = 0; j < conf.vLanguagePriority.size(); ++j)
- {
- std::cerr << "\t " << j << ": " << Util::getOptionNameString(conf.vLanguagePriority[j], GlobalConstants::LANGUAGES) << std::endl;
- }
- }
- if (conf.iInstallerPlatform != config.iInstallerPlatform)
- std::cerr << "\tPlatform: " << Util::getOptionNameString(conf.iInstallerPlatform, GlobalConstants::PLATFORMS) << std::endl;
- if (conf.vPlatformPriority != config.vPlatformPriority)
- {
- std::cerr << "\tPlatform priority:" << std::endl;
- for (unsigned int j = 0; j < conf.vPlatformPriority.size(); ++j)
- {
- std::cerr << "\t " << j << ": " << Util::getOptionNameString(conf.vPlatformPriority[j], GlobalConstants::PLATFORMS) << std::endl;
- }
- }
- }
- }
+ threads = gameItemQueue.size();
+ }
+ std::vector<std::thread> vThreads;
+ for (unsigned int i = 0; i < threads; ++i)
+ {
+ DownloadInfo dlInfo;
+ dlInfo.setStatus(DLSTATUS_NOTSTARTED);
+ vDownloadInfo.push_back(dlInfo);
+ vThreads.push_back(std::thread(Downloader::getGameDetailsThread, this->config, i));
}
- game = gogAPI->getGameDetails(gameItems[i].name, conf.iInstallerPlatform, conf.iInstallerLanguage, config.bDuplicateHandler);
- if (!gogAPI->getError())
+ unsigned int dl_status = DLSTATUS_NOTSTARTED;
+ while (dl_status != DLSTATUS_FINISHED)
{
- game.filterWithPriorities(conf);
- Json::Value gameDetailsJSON;
+ dl_status = DLSTATUS_NOTSTARTED;
- if (!gameItems[i].gamedetailsjson.empty())
- gameDetailsJSON = gameItems[i].gamedetailsjson;
+ // Print progress information once per 100ms
+ std::this_thread::sleep_for(std::chrono::milliseconds(100));
+ std::cerr << "\033[J\r" << std::flush; // Clear screen from the current line down to the bottom of the screen
- if (game.extras.empty() && config.bExtras) // Try to get extras from account page if API didn't return any extras
- {
- if (gameDetailsJSON.empty())
- gameDetailsJSON = gogWebsite->getGameDetailsJSON(gameItems[i].id);
- game.extras = this->getExtrasFromJSON(gameDetailsJSON, gameItems[i].name);
- }
- if (config.bSaveSerials)
- {
- if (gameDetailsJSON.empty())
- gameDetailsJSON = gogWebsite->getGameDetailsJSON(gameItems[i].id);
- game.serials = this->getSerialsFromJSON(gameDetailsJSON);
- }
- if (config.bSaveChangelogs)
+ // Print messages from message queue first
+ Message msg;
+ while (msgQueue.try_pop(msg))
{
- if (gameDetailsJSON.empty())
- gameDetailsJSON = gogWebsite->getGameDetailsJSON(gameItems[i].id);
- game.changelog = this->getChangelogFromJSON(gameDetailsJSON);
+ std::cerr << msg.getFormattedString(config.bColor, true) << std::endl;
+ if (config.bReport)
+ {
+ this->report_ofs << msg.getTimestampString() << ": " << msg.getMessage() << std::endl;
+ }
}
- // Ignore DLC count and try to get DLCs from JSON
- if (game.dlcs.empty() && !bHasDLC && conf.bDLC && conf.bIgnoreDLCCount)
+ for (unsigned int i = 0; i < vDownloadInfo.size(); ++i)
{
- if (gameDetailsJSON.empty())
- gameDetailsJSON = gogWebsite->getGameDetailsJSON(gameItems[i].id);
-
- gameItems[i].dlcnames = Util::getDLCNamesFromJSON(gameDetailsJSON["dlcs"]);
- bHasDLC = !gameItems[i].dlcnames.empty();
+ unsigned int status = vDownloadInfo[i].getStatus();
+ dl_status |= status;
}
- if (game.dlcs.empty() && bHasDLC && conf.bDLC)
- {
- for (unsigned int j = 0; j < gameItems[i].dlcnames.size(); ++j)
- {
- gameDetails dlc;
- dlc = gogAPI->getGameDetails(gameItems[i].dlcnames[j], conf.iInstallerPlatform, conf.iInstallerLanguage, config.bDuplicateHandler);
- dlc.filterWithPriorities(conf);
- if (dlc.extras.empty() && config.bExtras) // Try to get extras from account page if API didn't return any extras
- {
- if (gameDetailsJSON.empty())
- gameDetailsJSON = gogWebsite->getGameDetailsJSON(gameItems[i].id);
-
- // Make sure we get extras for the right DLC
- for (unsigned int k = 0; k < gameDetailsJSON["dlcs"].size(); ++k)
- {
- std::vector<std::string> urls;
- if (gameDetailsJSON["dlcs"][k].isMember("extras"))
- Util::getDownloaderUrlsFromJSON(gameDetailsJSON["dlcs"][k]["extras"], urls);
-
- if (!urls.empty())
- {
- if (urls[0].find("/" + gameItems[i].dlcnames[j] + "/") != std::string::npos)
- {
- dlc.extras = this->getExtrasFromJSON(gameDetailsJSON["dlcs"][k], gameItems[i].dlcnames[j]);
- }
- }
- }
- }
-
- if (config.bSaveSerials)
- {
- if (gameDetailsJSON.empty())
- gameDetailsJSON = gogWebsite->getGameDetailsJSON(gameItems[i].id);
-
- // Make sure we save serial for the right DLC
- for (unsigned int k = 0; k < gameDetailsJSON["dlcs"].size(); ++k)
- {
- std::vector<std::string> urls;
- if (gameDetailsJSON["dlcs"][k].isMember("cdKey") && gameDetailsJSON["dlcs"][k].isMember("downloads"))
- {
- // Assuming that only DLC with installers can have serial
- Util::getDownloaderUrlsFromJSON(gameDetailsJSON["dlcs"][k]["downloads"], urls);
- }
-
- if (!urls.empty())
- {
- if (urls[0].find("/" + gameItems[i].dlcnames[j] + "/") != std::string::npos)
- {
- dlc.serials = this->getSerialsFromJSON(gameDetailsJSON["dlcs"][k]);
- }
- }
- }
- }
-
- if (config.bSaveChangelogs)
- {
- if (gameDetailsJSON.empty())
- gameDetailsJSON = gogWebsite->getGameDetailsJSON(gameItems[i].id);
-
- // Make sure we save changelog for the right DLC
- for (unsigned int k = 0; k < gameDetailsJSON["dlcs"].size(); ++k)
- {
- std::vector<std::string> urls;
- if (gameDetailsJSON["dlcs"][k].isMember("changelog") && gameDetailsJSON["dlcs"][k].isMember("downloads"))
- {
- // Assuming that only DLC with installers can have changelog
- Util::getDownloaderUrlsFromJSON(gameDetailsJSON["dlcs"][k]["downloads"], urls);
- }
-
- if (!urls.empty())
- {
- if (urls[0].find("/" + gameItems[i].dlcnames[j] + "/") != std::string::npos)
- {
- dlc.changelog = this->getChangelogFromJSON(gameDetailsJSON["dlcs"][k]);
- }
- }
- }
- }
+ std::cerr << "Getting game info " << (gameItems.size() - gameItemQueue.size()) << " / " << gameItems.size() << std::endl;
- game.dlcs.push_back(dlc);
- }
+ if (dl_status != DLSTATUS_FINISHED)
+ {
+ std::cerr << "\033[1A\r" << std::flush; // Move cursor up by 1 row
}
+ }
- game.makeFilepaths(conf.dirConf);
+ // Join threads
+ for (unsigned int i = 0; i < vThreads.size(); ++i)
+ vThreads[i].join();
- if (!config.bUpdateCheck)
- games.push_back(game);
- else
- { // Update check, only add games that have updated files
- for (unsigned int j = 0; j < game.installers.size(); ++j)
- {
- if (game.installers[j].updated)
- {
- games.push_back(game);
- updated++;
- break; // add the game only once
- }
- }
- if (updated >= gogAPI->user.notifications_games)
- { // Gone through all updated games. No need to go through the rest.
- std::cerr << std::endl << "Got info for all updated games. Moving on..." << std::endl;
- break;
- }
- }
- }
- else
+ vThreads.clear();
+ vDownloadInfo.clear();
+
+ gameDetails details;
+ while (gameDetailsQueue.try_pop(details))
{
- std::cerr << gogAPI->getErrorMessage() << std::endl;
- gogAPI->clearError();
- continue;
+ this->games.push_back(details);
}
+ std::sort(this->games.begin(), this->games.end(), [](const gameDetails& i, const gameDetails& j) -> bool { return i.gamename < j.gamename; });
}
- std::cerr << std::endl;
+
return 0;
}
-void Downloader::listGames()
+int Downloader::listGames()
{
if (config.bListDetails) // Detailed list
{
- if (this->games.empty())
- this->getGameDetails();
+ if (this->games.empty()) {
+ int res = this->getGameDetails();
+ if (res > 0)
+ return res;
+ }
for (unsigned int i = 0; i < games.size(); ++i)
{
@@ -641,6 +555,22 @@ void Downloader::listGames()
<< "\tsize: " << games[i].dlcs[j].extras[k].size << std::endl
<< std::endl;
}
+ for (unsigned int k = 0; k < games[i].dlcs[j].languagepacks.size(); ++k)
+ {
+ std::string filepath = games[i].dlcs[j].languagepacks[k].getFilepath();
+ if (config.blacklist.isBlacklisted(filepath)) {
+ if (config.bVerbose)
+ std::cerr << "skipped blacklisted file " << filepath << std::endl;
+ continue;
+ }
+
+ std::cout << "\tgamename: " << games[i].dlcs[j].gamename << std::endl
+ << "\tid: " << games[i].dlcs[j].languagepacks[k].id << std::endl
+ << "\tname: " << games[i].dlcs[j].languagepacks[k].name << std::endl
+ << "\tpath: " << games[i].dlcs[j].languagepacks[k].path << std::endl
+ << "\tsize: " << games[i].dlcs[j].languagepacks[k].size << std::endl
+ << std::endl;
+ }
}
}
}
@@ -655,6 +585,7 @@ void Downloader::listGames()
}
}
+ return 0;
}
void Downloader::repair()
@@ -901,6 +832,41 @@ void Downloader::repair()
std::cout << std::endl;
}
}
+ if (config.bLanguagePacks)
+ {
+ for (unsigned int k = 0; k < games[i].dlcs[j].languagepacks.size(); ++k)
+ {
+ std::string filepath = games[i].dlcs[j].languagepacks[k].getFilepath();
+ if (config.blacklist.isBlacklisted(filepath)) {
+ if (config.bVerbose)
+ std::cerr << "skipped blacklisted file " << filepath << std::endl;
+ continue;
+ }
+
+ // Get XML data
+ std::string XML = "";
+ if (config.bRemoteXML)
+ {
+ XML = gogAPI->getXML(games[i].dlcs[j].gamename, games[i].dlcs[j].languagepacks[k].id);
+ if (gogAPI->getError())
+ {
+ std::cerr << gogAPI->getErrorMessage() << std::endl;
+ gogAPI->clearError();
+ }
+ }
+
+ std::string url = gogAPI->getLanguagePackLink(games[i].dlcs[j].gamename, games[i].dlcs[j].languagepacks[k].id);
+ if (gogAPI->getError())
+ {
+ std::cerr << gogAPI->getErrorMessage() << std::endl;
+ gogAPI->clearError();
+ continue;
+ }
+ std::cout << "Repairing file " << filepath << std::endl;
+ this->repairFile(url, filepath, XML, games[i].dlcs[j].gamename);
+ std::cout << std::endl;
+ }
+ }
}
}
}
@@ -931,332 +897,118 @@ void Downloader::download()
if (!games[i].installers.empty())
{
// Take path from installer path because for some games the base directory for installer/extra path is not "gamename"
- std::string filepath = games[i].installers[0].getFilepath();
+ boost::filesystem::path filepath = boost::filesystem::absolute(games[i].installers[0].getFilepath(), boost::filesystem::current_path());
// Get base directory from filepath
- boost::match_results<std::string::const_iterator> what;
- boost::regex expression("(.*)/.*");
- boost::regex_match(filepath, what, expression);
- std::string directory = what[1];
+ std::string directory = filepath.parent_path().string();
this->downloadCovers(games[i].gamename, directory, coverXML);
}
}
- // Download installers
+
if (config.bInstallers)
{
for (unsigned int j = 0; j < games[i].installers.size(); ++j)
{
- // Not updated, skip to next installer
- if (config.bUpdateCheck && !games[i].installers[j].updated)
- continue;
-
- std::string filepath = games[i].installers[j].getFilepath();
- if (config.blacklist.isBlacklisted(filepath))
+ dlQueue.push(games[i].installers[j]);
+ }
+ }
+ if (config.bPatches)
+ {
+ for (unsigned int j = 0; j < games[i].patches.size(); ++j)
+ {
+ dlQueue.push(games[i].patches[j]);
+ }
+ }
+ if (config.bExtras)
+ {
+ for (unsigned int j = 0; j < games[i].extras.size(); ++j)
+ {
+ dlQueue.push(games[i].extras[j]);
+ }
+ }
+ if (config.bLanguagePacks)
+ {
+ for (unsigned int j = 0; j < games[i].languagepacks.size(); ++j)
+ {
+ dlQueue.push(games[i].languagepacks[j]);
+ }
+ }
+ if (config.bDLC && !games[i].dlcs.empty())
+ {
+ for (unsigned int j = 0; j < games[i].dlcs.size(); ++j)
+ {
+ if (config.bSaveSerials && !games[i].dlcs[j].serials.empty())
{
- if (config.bVerbose)
- std::cerr << "skipped blacklisted file " << filepath << std::endl;
- continue;
+ std::string filepath = games[i].dlcs[j].getSerialsFilepath();
+ this->saveSerials(games[i].dlcs[j].serials, filepath);
}
-
- // Get link
- std::string url = gogAPI->getInstallerLink(games[i].gamename, games[i].installers[j].id);
- if (gogAPI->getError())
+ if (config.bSaveChangelogs && !games[i].dlcs[j].changelog.empty())
{
- std::cerr << gogAPI->getErrorMessage() << std::endl;
- gogAPI->clearError();
- continue;
+ std::string filepath = games[i].dlcs[j].getChangelogFilepath();
+ this->saveChangelog(games[i].dlcs[j].changelog, filepath);
}
- // Download
- if (!url.empty())
+ if (config.bInstallers)
{
- std::string XML;
- if (config.bRemoteXML)
+ for (unsigned int k = 0; k < games[i].dlcs[j].installers.size(); ++k)
{
- XML = gogAPI->getXML(games[i].gamename, games[i].installers[j].id);
- if (gogAPI->getError())
- {
- std::cerr << gogAPI->getErrorMessage() << std::endl;
- gogAPI->clearError();
- }
+ dlQueue.push(games[i].dlcs[j].installers[k]);
}
- if (!games[i].installers[j].name.empty())
- std::cout << "Downloading: " << games[i].installers[j].name << std::endl;
- std::cout << filepath << std::endl;
- this->downloadFile(url, filepath, XML, games[i].gamename);
- std::cout << std::endl;
}
- }
- }
- // Download extras
- if (config.bExtras && !config.bUpdateCheck)
- { // Save some time and don't process extras when running update check. Extras don't have updated flag, all of them would be skipped anyway.
- for (unsigned int j = 0; j < games[i].extras.size(); ++j)
- {
- // Get link
- std::string url = gogAPI->getExtraLink(games[i].gamename, games[i].extras[j].id);
- if (gogAPI->getError())
- {
- std::cerr << gogAPI->getErrorMessage() << std::endl;
- gogAPI->clearError();
- continue;
- }
-
- std::string filepath = games[i].extras[j].getFilepath();
- if (config.blacklist.isBlacklisted(filepath))
- {
- if (config.bVerbose)
- std::cerr << "skipped blacklisted file " << filepath << std::endl;
- continue;
- }
-
- // Download
- if (!url.empty())
+ if (config.bPatches)
{
- if (!games[i].extras[j].name.empty())
- std::cout << "Downloading: " << games[i].extras[j].name << std::endl;
- std::cout << filepath << std::endl;
- CURLcode result = this->downloadFile(url, filepath);
- std::cout << std::endl;
- if (result==CURLE_OK && config.bAutomaticXMLCreation)
+ for (unsigned int k = 0; k < games[i].dlcs[j].patches.size(); ++k)
{
- std::cout << "Starting automatic XML creation" << std::endl;
- std::string xml_dir = config.sXMLDirectory + "/" + games[i].gamename;
- Util::createXML(filepath, config.iChunkSize, xml_dir);
- std::cout << std::endl;
+ dlQueue.push(games[i].dlcs[j].patches[k]);
}
}
- }
- }
- // Download patches
- if (config.bPatches)
- {
- for (unsigned int j = 0; j < games[i].patches.size(); ++j)
- {
- // Not updated, skip to next patch
- if (config.bUpdateCheck && !games[i].patches[j].updated)
- continue;
-
- // Get link
- std::string url = gogAPI->getPatchLink(games[i].gamename, games[i].patches[j].id);
- if (gogAPI->getError())
- {
- std::cerr << gogAPI->getErrorMessage() << std::endl;
- gogAPI->clearError();
- continue;
- }
-
- std::string filepath = games[i].patches[j].getFilepath();
- if (config.blacklist.isBlacklisted(filepath))
- {
- if (config.bVerbose)
- std::cerr << "skipped blacklisted file " << filepath << std::endl;
- continue;
- }
-
- // Download
- if (!url.empty())
+ if (config.bExtras)
{
- std::string XML;
- if (config.bRemoteXML)
+ for (unsigned int k = 0; k < games[i].dlcs[j].extras.size(); ++k)
{
- XML = gogAPI->getXML(games[i].gamename, games[i].patches[j].id);
- if (gogAPI->getError())
- {
- std::cerr << gogAPI->getErrorMessage() << std::endl;
- gogAPI->clearError();
- }
+ dlQueue.push(games[i].dlcs[j].extras[k]);
}
- if (!games[i].patches[j].name.empty())
- std::cout << "Downloading: " << games[i].patches[j].name << std::endl;
- std::cout << filepath << std::endl;
- this->downloadFile(url, filepath, XML, games[i].gamename);
- std::cout << std::endl;
- }
- }
- }
- // Download language packs
- if (config.bLanguagePacks)
- {
- for (unsigned int j = 0; j < games[i].languagepacks.size(); ++j)
- {
- // Get link
- std::string url = gogAPI->getLanguagePackLink(games[i].gamename, games[i].languagepacks[j].id);
- if (gogAPI->getError())
- {
- std::cerr << gogAPI->getErrorMessage() << std::endl;
- gogAPI->clearError();
- continue;
- }
-
- std::string filepath = games[i].languagepacks[j].getFilepath();
- if (config.blacklist.isBlacklisted(filepath))
- {
- if (config.bVerbose)
- std::cerr << "skipped blacklisted file " << filepath << std::endl;
- continue;
}
-
- // Download
- if (!url.empty())
+ if (config.bLanguagePacks)
{
- std::string XML;
- if (config.bRemoteXML)
+ for (unsigned int k = 0; k < games[i].dlcs[j].languagepacks.size(); ++k)
{
- XML = gogAPI->getXML(games[i].gamename, games[i].languagepacks[j].id);
- if (gogAPI->getError())
- {
- std::cerr << gogAPI->getErrorMessage() << std::endl;
- gogAPI->clearError();
- }
+ dlQueue.push(games[i].dlcs[j].languagepacks[k]);
}
- if (!games[i].languagepacks[j].name.empty())
- std::cout << "Downloading: " << games[i].gamename << " " << games[i].languagepacks[j].name << std::endl;
- std::cout << filepath << std::endl;
- this->downloadFile(url, filepath, XML, games[i].gamename);
- std::cout << std::endl;
}
}
}
- if (config.bDLC && !games[i].dlcs.empty())
- {
- for (unsigned int j = 0; j < games[i].dlcs.size(); ++j)
- {
- if (config.bSaveSerials && !games[i].dlcs[j].serials.empty())
- {
- std::string filepath = games[i].dlcs[j].getSerialsFilepath();
- this->saveSerials(games[i].dlcs[j].serials, filepath);
- }
- if (config.bSaveChangelogs && !games[i].dlcs[j].changelog.empty())
- {
- std::string filepath = games[i].dlcs[j].getChangelogFilepath();
- this->saveChangelog(games[i].dlcs[j].changelog, filepath);
- }
-
- if (config.bInstallers)
- {
- for (unsigned int k = 0; k < games[i].dlcs[j].installers.size(); ++k)
- {
- std::string filepath = games[i].dlcs[j].installers[k].getFilepath();
- if (config.blacklist.isBlacklisted(filepath))
- {
- if (config.bVerbose)
- std::cerr << "skipped blacklisted file " << filepath << std::endl;
- continue;
- }
-
- // Get link
- std::string url = gogAPI->getInstallerLink(games[i].dlcs[j].gamename, games[i].dlcs[j].installers[k].id);
- if (gogAPI->getError())
- {
- std::cerr << gogAPI->getErrorMessage() << std::endl;
- gogAPI->clearError();
- continue;
- }
+ }
- // Download
- if (!url.empty())
- {
- std::string XML;
- if (config.bRemoteXML)
- {
- XML = gogAPI->getXML(games[i].dlcs[j].gamename, games[i].dlcs[j].installers[k].id);
- if (gogAPI->getError())
- {
- std::cerr << gogAPI->getErrorMessage() << std::endl;
- gogAPI->clearError();
- }
- }
- if (!games[i].dlcs[j].installers[k].name.empty())
- std::cout << "Downloading: " << games[i].dlcs[j].installers[k].name << std::endl;
- std::cout << filepath << std::endl;
- this->downloadFile(url, filepath, XML, games[i].dlcs[j].gamename);
- std::cout << std::endl;
- }
- }
- }
- if (config.bPatches)
- {
- for (unsigned int k = 0; k < games[i].dlcs[j].patches.size(); ++k)
- {
- std::string filepath = games[i].dlcs[j].patches[k].getFilepath();
- if (config.blacklist.isBlacklisted(filepath))
- {
- if (config.bVerbose)
- std::cerr << "skipped blacklisted file " << filepath << std::endl;
- continue;
- }
+ // Create download threads
+ std::vector<std::thread> vThreads;
+ for (unsigned int i = 0; i < config.iThreads; ++i)
+ {
+ DownloadInfo dlInfo;
+ dlInfo.setStatus(DLSTATUS_NOTSTARTED);
+ vDownloadInfo.push_back(dlInfo);
+ vThreads.push_back(std::thread(Downloader::processDownloadQueue, this->config, i));
+ }
- // Get link
- std::string url = gogAPI->getPatchLink(games[i].dlcs[j].gamename, games[i].dlcs[j].patches[k].id);
- if (gogAPI->getError())
- {
- std::cerr << gogAPI->getErrorMessage() << std::endl;
- gogAPI->clearError();
- continue;
- }
+ this->printProgress();
- // Download
- if (!url.empty())
- {
- std::string XML;
- if (config.bRemoteXML)
- {
- XML = gogAPI->getXML(games[i].dlcs[j].gamename, games[i].dlcs[j].patches[k].id);
- if (gogAPI->getError())
- {
- std::cerr << gogAPI->getErrorMessage() << std::endl;
- gogAPI->clearError();
- }
- }
- if (!games[i].dlcs[j].patches[k].name.empty())
- std::cout << "Downloading: " << games[i].dlcs[j].patches[k].name << std::endl;
- std::cout << filepath << std::endl;
- this->downloadFile(url, filepath, XML, games[i].dlcs[j].gamename);
- std::cout << std::endl;
- }
- }
- }
- if (config.bExtras)
- {
- for (unsigned int k = 0; k < games[i].dlcs[j].extras.size(); ++k)
- {
- std::string filepath = games[i].dlcs[j].extras[k].getFilepath();
- if (config.blacklist.isBlacklisted(filepath))
- {
- if (config.bVerbose)
- std::cerr << "skipped blacklisted file " << filepath << std::endl;
- continue;
- }
+ // Join threads
+ for (unsigned int i = 0; i < vThreads.size(); ++i)
+ vThreads[i].join();
- // Get link
- std::string url = gogAPI->getExtraLink(games[i].dlcs[j].gamename, games[i].dlcs[j].extras[k].id);
- if (gogAPI->getError())
- {
- std::cerr << gogAPI->getErrorMessage() << std::endl;
- gogAPI->clearError();
- continue;
- }
+ vThreads.clear();
+ vDownloadInfo.clear();
- // Download
- if (!url.empty())
- {
- if (!games[i].dlcs[j].extras[k].name.empty())
- std::cout << "Dowloading: " << games[i].dlcs[j].extras[k].name << std::endl;
- CURLcode result = this->downloadFile(url, filepath);
- std::cout << std::endl;
- if (result==CURLE_OK && config.bAutomaticXMLCreation)
- {
- std::cout << "Starting automatic XML creation" << std::endl;
- std::string xml_dir = config.sXMLDirectory + "/" + games[i].dlcs[j].gamename;
- Util::createXML(filepath, config.iChunkSize, xml_dir);
- std::cout << std::endl;
- }
- }
- }
- }
- }
+ // Create xml data for all files in the queue
+ if (!createXMLQueue.empty())
+ {
+ std::cout << "Starting XML creation" << std::endl;
+ gameFile gf;
+ while (createXMLQueue.try_pop(gf))
+ {
+ std::string xml_directory = config.sXMLDirectory + "/" + gf.gamename;
+ Util::createXML(gf.getFilepath(), config.iChunkSize, xml_directory);
}
}
}
@@ -1293,12 +1045,11 @@ CURLcode Downloader::downloadFile(const std::string& url, const std::string& fil
// Do version check if local hash exists
if (!localHash.empty())
{
- TiXmlDocument remote_xml;
+ tinyxml2::XMLDocument remote_xml;
remote_xml.Parse(xml_data.c_str());
- TiXmlNode *fileNodeRemote = remote_xml.FirstChild("file");
- if (fileNodeRemote)
+ tinyxml2::XMLElement *fileElemRemote = remote_xml.FirstChildElement("file");
+ if (fileElemRemote)
{
- TiXmlElement *fileElemRemote = fileNodeRemote->ToElement();
std::string remoteHash = fileElemRemote->Attribute("md5");
if (remoteHash != localHash)
bSameVersion = false;
@@ -1496,7 +1247,7 @@ int Downloader::repairFile(const std::string& url, const std::string& filepath,
bool bFileExists = boost::filesystem::exists(pathname);
bool bLocalXMLExists = boost::filesystem::exists(xml_file);
- TiXmlDocument xml;
+ tinyxml2::XMLDocument xml;
if (!xml_data.empty()) // Parse remote XML data
{
std::cout << "XML: Using remote file" << std::endl;
@@ -1507,12 +1258,12 @@ int Downloader::repairFile(const std::string& url, const std::string& filepath,
std::cout << "XML: Using local file" << std::endl;
if (!bLocalXMLExists)
std::cout << "XML: File doesn't exist (" << xml_file << ")" << std::endl;
- xml.LoadFile(xml_file);
+ xml.LoadFile(xml_file.c_str());
}
// Check if file node exists in XML data
- TiXmlNode *fileNode = xml.FirstChild("file");
- if (!fileNode)
+ tinyxml2::XMLElement *fileElem = xml.FirstChildElement("file");
+ if (!fileElem)
{ // File node doesn't exist
std::cout << "XML: Parsing failed / not valid XML" << std::endl;
if (config.bDownload)
@@ -1523,23 +1274,21 @@ int Downloader::repairFile(const std::string& url, const std::string& filepath,
else
{ // File node exists --> valid XML
std::cout << "XML: Valid XML" << std::endl;
- TiXmlElement *fileElem = fileNode->ToElement();
filename = fileElem->Attribute("name");
filehash = fileElem->Attribute("md5");
std::stringstream(fileElem->Attribute("chunks")) >> chunks;
std::stringstream(fileElem->Attribute("total_size")) >> filesize;
//Iterate through all chunk nodes
- TiXmlNode *chunkNode = fileNode->FirstChild();
- while (chunkNode)
+ tinyxml2::XMLElement *chunkElem = fileElem->FirstChildElement("chunk");
+ while (chunkElem)
{
- TiXmlElement *chunkElem = chunkNode->ToElement();
std::stringstream(chunkElem->Attribute("from")) >> from_offset;
std::stringstream(chunkElem->Attribute("to")) >> to_offset;
chunk_from.push_back(from_offset);
chunk_to.push_back(to_offset);
chunk_hash.push_back(chunkElem->GetText());
- chunkNode = fileNode->IterateChildren(chunkNode);
+ chunkElem = chunkElem->NextSiblingElement("chunk");
}
std::cout << "XML: Parsing finished" << std::endl << std::endl
@@ -1735,7 +1484,7 @@ int Downloader::repairFile(const std::string& url, const std::string& filepath,
int Downloader::downloadCovers(const std::string& gamename, const std::string& directory, const std::string& cover_xml_data)
{
int res = 0;
- TiXmlDocument xml;
+ tinyxml2::XMLDocument xml;
// Check that directory exists and create subdirectories
boost::filesystem::path path = directory;
@@ -1758,7 +1507,7 @@ int Downloader::downloadCovers(const std::string& gamename, const std::string& d
}
xml.Parse(cover_xml_data.c_str());
- TiXmlElement *rootNode = xml.RootElement();
+ tinyxml2::XMLElement *rootNode = xml.RootElement();
if (!rootNode)
{
std::cout << "Not valid XML" << std::endl;
@@ -1766,19 +1515,19 @@ int Downloader::downloadCovers(const std::string& gamename, const std::string& d
}
else
{
- TiXmlNode *gameNode = rootNode->FirstChild();
+ tinyxml2::XMLNode *gameNode = rootNode->FirstChild();
while (gameNode)
{
- TiXmlElement *gameElem = gameNode->ToElement();
+ tinyxml2::XMLElement *gameElem = gameNode->ToElement();
std::string game_name = gameElem->Attribute("name");
if (game_name == gamename)
{
boost::match_results<std::string::const_iterator> what;
- TiXmlNode *coverNode = gameNode->FirstChild();
+ tinyxml2::XMLNode *coverNode = gameNode->FirstChild();
while (coverNode)
{
- TiXmlElement *coverElem = coverNode->ToElement();
+ tinyxml2::XMLElement *coverElem = coverNode->ToElement();
std::string cover_url = coverElem->GetText();
// Get file extension for the image
boost::regex e1(".*(\\.\\w+)$", boost::regex::perl | boost::regex::icase);
@@ -1806,11 +1555,11 @@ int Downloader::downloadCovers(const std::string& gamename, const std::string& d
std::cout << "failed to get error code: " << curl_easy_strerror(result) << " (" << cover_url << ")" << std::endl;
}
- coverNode = gameNode->IterateChildren(coverNode);
+ coverNode = coverNode->NextSibling();
}
break; // Found cover for game, no need to go through rest of the game nodes
}
- gameNode = rootNode->IterateChildren(gameNode);
+ gameNode = gameNode->NextSibling();
}
}
@@ -1871,6 +1620,10 @@ std::string Downloader::getResponse(const std::string& url)
int Downloader::progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
{
+ // unused so lets prevent warnings and be more pedantic
+ (void) ulnow;
+ (void) ultotal;
+
// on entry: dltotal - how much remains to download till the end of the file (bytes)
// dlnow - how much was downloaded from the start of the program (bytes)
int bar_length = 26;
@@ -2007,12 +1760,23 @@ uintmax_t Downloader::getResumePosition()
return this->resume_position;
}
-std::vector<gameFile> Downloader::getExtrasFromJSON(const Json::Value& json, const std::string& gamename)
+std::vector<gameFile> Downloader::getExtrasFromJSON(const Json::Value& json, const std::string& gamename, const Config& config)
{
std::vector<gameFile> extras;
- std::vector<std::string> downloaderUrls;
- Util::getDownloaderUrlsFromJSON(json["extras"], downloaderUrls);
+ // Create new API handle and set curl options for the API
+ API* api = new API(config.sToken, config.sSecret);
+ api->curlSetOpt(CURLOPT_VERBOSE, config.bVerbose);
+ api->curlSetOpt(CURLOPT_SSL_VERIFYPEER, config.bVerifyPeer);
+ api->curlSetOpt(CURLOPT_CONNECTTIMEOUT, config.iTimeout);
+ if (!config.sCACertPath.empty())
+ api->curlSetOpt(CURLOPT_CAINFO, config.sCACertPath.c_str());
+
+ if (!api->init())
+ {
+ delete api;
+ return extras;
+ }
for (unsigned int i = 0; i < json["extras"].size(); ++i)
{
@@ -2022,7 +1786,12 @@ std::vector<gameFile> Downloader::getExtrasFromJSON(const Json::Value& json, con
id.assign(downloaderUrl.begin()+downloaderUrl.find_last_of("/")+1, downloaderUrl.end());
// Get path from download link
- std::string url = gogAPI->getExtraLink(gamename, id);
+ std::string url = api->getExtraLink(gamename, id);
+ if (api->getError())
+ {
+ api->clearError();
+ continue;
+ }
url = htmlcxx::Uri::decode(url);
if (url.find("/extras/") != std::string::npos)
{
@@ -2071,6 +1840,7 @@ std::vector<gameFile> Downloader::getExtrasFromJSON(const Json::Value& json, con
extras.push_back(gf);
}
+ delete api;
return extras;
}
@@ -2253,11 +2023,12 @@ void Downloader::checkOrphans()
{ // Check dlcs
for (unsigned int k = 0; k < games[i].dlcs.size(); ++k)
{
- bFoundFile = isPresent(games[i].dlcs[k].installers, filepath_vector[j], config.blacklist)
- || isPresent(games[i].dlcs[k].extras, filepath_vector[j], config.blacklist)
- || isPresent(games[i].dlcs[k].patches, filepath_vector[j], config.blacklist);
- if(bFoundFile)
- break;
+ bFoundFile = isPresent(games[i].dlcs[k].installers, filepath_vector[j], config.blacklist)
+ || isPresent(games[i].dlcs[k].extras, filepath_vector[j], config.blacklist)
+ || isPresent(games[i].dlcs[k].patches, filepath_vector[j], config.blacklist)
+ || isPresent(games[i].dlcs[k].languagepacks, filepath_vector[j], config.blacklist);
+ if(bFoundFile)
+ break;
}
}
if (!bFoundFile)
@@ -2296,8 +2067,8 @@ void Downloader::checkStatus()
{
boost::filesystem::path filepath = games[i].installers[j].getFilepath();
- if (config.blacklist.isBlacklisted(filepath.native()))
- continue;
+ if (config.blacklist.isBlacklisted(filepath.native()))
+ continue;
std::string remoteHash;
std::string localHash;
bool bHashOK = true; // assume hash OK
@@ -2326,12 +2097,11 @@ void Downloader::checkStatus()
if (boost::filesystem::exists(local_xml_file))
{
- TiXmlDocument local_xml;
- local_xml.LoadFile(local_xml_file.string());
- TiXmlNode *fileNodeLocal = local_xml.FirstChild("file");
- if (fileNodeLocal)
+ tinyxml2::XMLDocument local_xml;
+ local_xml.LoadFile(local_xml_file.string().c_str());
+ tinyxml2::XMLElement *fileElemLocal = local_xml.FirstChildElement("file");
+ if (fileElemLocal)
{
- TiXmlElement *fileElemLocal = fileNodeLocal->ToElement();
std::string filesize_xml_str = fileElemLocal->Attribute("total_size");
filesize_xml = std::stoull(filesize_xml_str);
}
@@ -2360,8 +2130,8 @@ void Downloader::checkStatus()
{
boost::filesystem::path filepath = games[i].extras[j].getFilepath();
- if (config.blacklist.isBlacklisted(filepath.native()))
- continue;
+ if (config.blacklist.isBlacklisted(filepath.native()))
+ continue;
std::string localHash = this->getLocalFileHash(filepath.string(), games[i].gamename);
uintmax_t filesize;
@@ -2383,8 +2153,8 @@ void Downloader::checkStatus()
{
boost::filesystem::path filepath = games[i].patches[j].getFilepath();
- if (config.blacklist.isBlacklisted(filepath.native()))
- continue;
+ if (config.blacklist.isBlacklisted(filepath.native()))
+ continue;
std::string localHash = this->getLocalFileHash(filepath.string(), games[i].gamename);
uintmax_t filesize;
@@ -2406,8 +2176,8 @@ void Downloader::checkStatus()
{
boost::filesystem::path filepath = games[i].languagepacks[j].getFilepath();
- if (config.blacklist.isBlacklisted(filepath.native()))
- continue;
+ if (config.blacklist.isBlacklisted(filepath.native()))
+ continue;
std::string localHash = this->getLocalFileHash(filepath.string(), games[i].gamename);
uintmax_t filesize;
@@ -2433,8 +2203,8 @@ void Downloader::checkStatus()
{
boost::filesystem::path filepath = games[i].dlcs[j].installers[k].getFilepath();
- if (config.blacklist.isBlacklisted(filepath.native()))
- continue;
+ if (config.blacklist.isBlacklisted(filepath.native()))
+ continue;
std::string remoteHash;
std::string localHash;
bool bHashOK = true; // assume hash OK
@@ -2463,12 +2233,11 @@ void Downloader::checkStatus()
if (boost::filesystem::exists(local_xml_file))
{
- TiXmlDocument local_xml;
- local_xml.LoadFile(local_xml_file.string());
- TiXmlNode *fileNodeLocal = local_xml.FirstChild("file");
- if (fileNodeLocal)
+ tinyxml2::XMLDocument local_xml;
+ local_xml.LoadFile(local_xml_file.string().c_str());
+ tinyxml2::XMLElement *fileElemLocal = local_xml.FirstChildElement("file");
+ if (fileElemLocal)
{
- TiXmlElement *fileElemLocal = fileNodeLocal->ToElement();
std::string filesize_xml_str = fileElemLocal->Attribute("total_size");
filesize_xml = std::stoull(filesize_xml_str);
}
@@ -2497,8 +2266,8 @@ void Downloader::checkStatus()
{
boost::filesystem::path filepath = games[i].dlcs[j].patches[k].getFilepath();
- if (config.blacklist.isBlacklisted(filepath.native()))
- continue;
+ if (config.blacklist.isBlacklisted(filepath.native()))
+ continue;
std::string localHash = this->getLocalFileHash(filepath.string(), games[i].dlcs[j].gamename);
uintmax_t filesize;
@@ -2520,8 +2289,31 @@ void Downloader::checkStatus()
{
boost::filesystem::path filepath = games[i].dlcs[j].extras[k].getFilepath();
- if (config.blacklist.isBlacklisted(filepath.native()))
- continue;
+ if (config.blacklist.isBlacklisted(filepath.native()))
+ continue;
+ std::string localHash = this->getLocalFileHash(filepath.string(), games[i].dlcs[j].gamename);
+ uintmax_t filesize;
+
+ if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath))
+ {
+ filesize = boost::filesystem::file_size(filepath);
+ std::cout << "OK " << games[i].dlcs[j].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl;
+ }
+ else
+ {
+ std::cout << "ND " << games[i].dlcs[j].gamename << " " << filepath.filename().string() << std::endl;
+ }
+ }
+ }
+
+ if (config.bLanguagePacks)
+ {
+ for (unsigned int k = 0; k < games[i].dlcs[j].languagepacks.size(); ++k)
+ {
+ boost::filesystem::path filepath = games[i].dlcs[j].languagepacks[k].getFilepath();
+
+ if (config.blacklist.isBlacklisted(filepath.native()))
+ continue;
std::string localHash = this->getLocalFileHash(filepath.string(), games[i].dlcs[j].gamename);
uintmax_t filesize;
@@ -2555,34 +2347,12 @@ std::string Downloader::getLocalFileHash(const std::string& filepath, const std:
if (config.bAutomaticXMLCreation && !boost::filesystem::exists(local_xml_file) && boost::filesystem::exists(path))
{
- std::string xml_directory = config.sXMLDirectory + "/" + gamename;
- Util::createXML(filepath, config.iChunkSize, xml_directory);
+ std::string xml_directory = config.sXMLDirectory + "/" + gamename;
+ Util::createXML(filepath, config.iChunkSize, xml_directory);
}
- if (boost::filesystem::exists(local_xml_file))
- {
- TiXmlDocument local_xml;
- local_xml.LoadFile(local_xml_file.string());
- TiXmlNode *fileNodeLocal = local_xml.FirstChild("file");
- if (!fileNodeLocal && config.bAutomaticXMLCreation)
- {
- std::string xml_directory = config.sXMLDirectory + "/" + gamename;
- Util::createXML(filepath, config.iChunkSize, xml_directory);
- local_xml.LoadFile(local_xml_file.string());
- fileNodeLocal = local_xml.FirstChild("file");
- }
- if (fileNodeLocal)
- {
- TiXmlElement *fileElemLocal = fileNodeLocal->ToElement();
- localHash = fileElemLocal->Attribute("md5");
- return localHash;
- }
- }
+ localHash = Util::getLocalFileHash(config.sXMLDirectory, filepath, gamename);
- if (boost::filesystem::exists(path) && boost::filesystem::is_regular_file(path))
- {
- localHash = Util::getFileHash(path.string(), RHASH_MD5);
- }
return localHash;
}
@@ -2597,12 +2367,11 @@ std::string Downloader::getRemoteFileHash(const std::string& gamename, const std
}
if (!xml_data.empty())
{
- TiXmlDocument remote_xml;
+ tinyxml2::XMLDocument remote_xml;
remote_xml.Parse(xml_data.c_str());
- TiXmlNode *fileNodeRemote = remote_xml.FirstChild("file");
- if (fileNodeRemote)
+ tinyxml2::XMLElement *fileElemRemote = remote_xml.FirstChildElement("file");
+ if (fileElemRemote)
{
- TiXmlElement *fileElemRemote = fileNodeRemote->ToElement();
remoteHash = fileElemRemote->Attribute("md5");
}
}
@@ -2928,8 +2697,9 @@ void Downloader::saveChangelog(const std::string& changelog, const std::string&
return;
}
-void Downloader::downloadFileWithId(const std::string& fileid_string, const std::string& output_filepath)
+int Downloader::downloadFileWithId(const std::string& fileid_string, const std::string& output_filepath)
{
+ int res = 1;
size_t pos = fileid_string.find("/");
if (pos == std::string::npos)
{
@@ -2963,7 +2733,7 @@ void Downloader::downloadFileWithId(const std::string& fileid_string, const std:
else
filepath = output_filepath;
std::cout << "Downloading: " << filepath << std::endl;
- this->downloadFile(url, filepath, std::string(), gamename);
+ res = this->downloadFile(url, filepath, std::string(), gamename);
std::cout << std::endl;
}
else
@@ -2973,7 +2743,7 @@ void Downloader::downloadFileWithId(const std::string& fileid_string, const std:
}
}
- return;
+ return res;
}
void Downloader::showWishlist()
@@ -3012,3 +2782,771 @@ void Downloader::showWishlist()
return;
}
+
+void Downloader::processDownloadQueue(Config conf, const unsigned int& tid)
+{
+ std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]";
+
+ API* api = new API(conf.sToken, conf.sSecret);
+ api->curlSetOpt(CURLOPT_SSL_VERIFYPEER, conf.bVerifyPeer);
+ api->curlSetOpt(CURLOPT_CONNECTTIMEOUT, conf.iTimeout);
+ if (!conf.sCACertPath.empty())
+ api->curlSetOpt(CURLOPT_CAINFO, conf.sCACertPath.c_str());
+
+ if (!api->init())
+ {
+ delete api;
+ msgQueue.push(Message("API init failed", MSGTYPE_ERROR, msg_prefix));
+ vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED);
+ return;
+ }
+
+ CURL* dlhandle = curl_easy_init();
+ curl_easy_setopt(dlhandle, CURLOPT_FOLLOWLOCATION, 1);
+ curl_easy_setopt(dlhandle, CURLOPT_USERAGENT, conf.sVersionString.c_str());
+ curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
+
+ curl_easy_setopt(dlhandle, CURLOPT_CONNECTTIMEOUT, conf.iTimeout);
+ curl_easy_setopt(dlhandle, CURLOPT_FAILONERROR, true);
+ curl_easy_setopt(dlhandle, CURLOPT_SSL_VERIFYPEER, conf.bVerifyPeer);
+ curl_easy_setopt(dlhandle, CURLOPT_VERBOSE, conf.bVerbose);
+ curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
+ curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData);
+ curl_easy_setopt(dlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, conf.iDownloadRate);
+
+ // Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds
+ curl_easy_setopt(dlhandle, CURLOPT_LOW_SPEED_TIME, 30);
+ curl_easy_setopt(dlhandle, CURLOPT_LOW_SPEED_LIMIT, 200);
+
+ if (!conf.sCACertPath.empty())
+ curl_easy_setopt(dlhandle, CURLOPT_CAINFO, conf.sCACertPath.c_str());
+
+ xferInfo xferinfo;
+ xferinfo.tid = tid;
+ xferinfo.curlhandle = dlhandle;
+
+ curl_easy_setopt(dlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallbackForThread);
+ curl_easy_setopt(dlhandle, CURLOPT_XFERINFODATA, &xferinfo);
+
+ gameFile gf;
+ while (dlQueue.try_pop(gf))
+ {
+ CURLcode result = CURLE_RECV_ERROR; // assume network error
+ int iRetryCount = 0;
+ off_t iResumePosition = 0;
+
+ vDownloadInfo[tid].setStatus(DLSTATUS_STARTING);
+
+ // Get directory from filepath
+ boost::filesystem::path filepath = gf.getFilepath();
+ filepath = boost::filesystem::absolute(filepath, boost::filesystem::current_path());
+ boost::filesystem::path directory = filepath.parent_path();
+
+ // Skip blacklisted files
+ if (conf.blacklist.isBlacklisted(filepath.string()))
+ {
+ msgQueue.push(Message("Blacklisted file: " + filepath.string(), MSGTYPE_INFO, msg_prefix));
+ continue;
+ }
+
+ std::string filenameXML = filepath.filename().string() + ".xml";
+ std::string xml_directory = conf.sXMLDirectory + "/" + gf.gamename;
+ boost::filesystem::path local_xml_file = xml_directory + "/" + filenameXML;
+
+ vDownloadInfo[tid].setFilename(filepath.filename().string());
+ msgQueue.push(Message("Begin download: " + filepath.filename().string(), MSGTYPE_INFO, msg_prefix));
+
+ // Check that directory exists and create subdirectories
+ mtx_create_directories.lock(); // Use mutex to avoid possible race conditions
+ if (boost::filesystem::exists(directory))
+ {
+ if (!boost::filesystem::is_directory(directory))
+ {
+ mtx_create_directories.unlock();
+ msgQueue.push(Message(directory.string() + " is not directory, skipping file (" + filepath.filename().string() + ")", MSGTYPE_WARNING, msg_prefix));
+ continue;
+ }
+ else
+ {
+ mtx_create_directories.unlock();
+ }
+ }
+ else
+ {
+ if (!boost::filesystem::create_directories(directory))
+ {
+ mtx_create_directories.unlock();
+ msgQueue.push(Message("Failed to create directory (" + directory.string() + "), skipping file (" + filepath.filename().string() + ")", MSGTYPE_ERROR, msg_prefix));
+ continue;
+ }
+ else
+ {
+ mtx_create_directories.unlock();
+ }
+ }
+
+ bool bSameVersion = true; // assume same version
+ bool bLocalXMLExists = boost::filesystem::exists(local_xml_file); // This is additional check to see if remote xml should be saved to speed up future version checks
+
+ std::string xml;
+ if (gf.type & (GFTYPE_INSTALLER | GFTYPE_PATCH) && conf.bRemoteXML)
+ {
+ xml = api->getXML(gf.gamename, gf.id);
+ if (api->getError())
+ {
+ msgQueue.push(Message(api->getErrorMessage(), MSGTYPE_ERROR, msg_prefix));
+ api->clearError();
+ }
+ else
+ {
+ if (!xml.empty())
+ {
+ std::string localHash = Util::getLocalFileHash(conf.sXMLDirectory, filepath.string(), gf.gamename);
+ // Do version check if local hash exists
+ if (!localHash.empty())
+ {
+ tinyxml2::XMLDocument remote_xml;
+ remote_xml.Parse(xml.c_str());
+ tinyxml2::XMLElement *fileElem = remote_xml.FirstChildElement("file");
+ if (fileElem)
+ {
+ std::string remoteHash = fileElem->Attribute("md5");
+ if (remoteHash != localHash)
+ bSameVersion = false;
+ }
+ }
+ }
+ }
+ }
+
+ bool bResume = false;
+ if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath))
+ {
+ if (bSameVersion)
+ {
+ bResume = true;
+ }
+ else
+ {
+ msgQueue.push(Message("Remote file is different, renaming local file", MSGTYPE_INFO, msg_prefix));
+ std::string date_old = "." + bptime::to_iso_string(bptime::second_clock::local_time()) + ".old";
+ boost::filesystem::path new_name = filepath.string() + date_old; // Rename old file by appending date and ".old" to filename
+ boost::system::error_code ec;
+ boost::filesystem::rename(filepath, new_name, ec); // Rename the file
+ if (ec)
+ {
+ msgQueue.push(Message("Failed to rename " + filepath.string() + " to " + new_name.string() + " - Skipping file", MSGTYPE_WARNING, msg_prefix));
+ continue;
+ }
+ }
+ }
+
+ // Save remote XML
+ if (!xml.empty())
+ {
+ if ((bLocalXMLExists && !bSameVersion) || !bLocalXMLExists)
+ {
+ // Check that directory exists and create subdirectories
+ boost::filesystem::path path = xml_directory;
+ mtx_create_directories.lock(); // Use mutex to avoid race conditions
+ if (boost::filesystem::exists(path))
+ {
+ if (!boost::filesystem::is_directory(path))
+ {
+ msgQueue.push(Message(path.string() + " is not directory", MSGTYPE_WARNING, msg_prefix));
+ }
+ }
+ else
+ {
+ if (!boost::filesystem::create_directories(path))
+ {
+ msgQueue.push(Message("Failed to create directory: " + path.string(), MSGTYPE_ERROR, msg_prefix));
+ }
+ }
+ mtx_create_directories.unlock();
+ std::ofstream ofs(local_xml_file.string().c_str());
+ if (ofs)
+ {
+ ofs << xml;
+ ofs.close();
+ }
+ else
+ {
+ msgQueue.push(Message("Can't create " + local_xml_file.string(), MSGTYPE_ERROR, msg_prefix));
+ }
+ }
+ }
+
+ // Get download url
+ std::string url;
+ if (gf.type == GFTYPE_INSTALLER)
+ url = api->getInstallerLink(gf.gamename, gf.id);
+ else if (gf.type == GFTYPE_PATCH)
+ url = api->getPatchLink(gf.gamename, gf.id);
+ else if (gf.type == GFTYPE_LANGPACK)
+ url = api->getLanguagePackLink(gf.gamename, gf.id);
+ else if (gf.type == GFTYPE_EXTRA)
+ url = api->getExtraLink(gf.gamename, gf.id);
+ else
+ url = api->getExtraLink(gf.gamename, gf.id); // assume extra if type didn't match any of the others
+
+ if (api->getError())
+ {
+ msgQueue.push(Message(api->getErrorMessage(), MSGTYPE_ERROR, msg_prefix));
+ api->clearError();
+ continue;
+ }
+
+ curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str());
+ do
+ {
+ if (iRetryCount != 0)
+ msgQueue.push(Message("Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + filepath.filename().string(), MSGTYPE_INFO, msg_prefix));
+
+ FILE* outfile;
+ // File exists, resume
+ if (bResume)
+ {
+ iResumePosition = boost::filesystem::file_size(filepath);
+ if ((outfile=fopen(filepath.string().c_str(), "r+"))!=NULL)
+ {
+ fseek(outfile, 0, SEEK_END);
+ curl_easy_setopt(dlhandle, CURLOPT_RESUME_FROM_LARGE, iResumePosition);
+ curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, outfile);
+ }
+ else
+ {
+ msgQueue.push(Message("Failed to open " + filepath.string(), MSGTYPE_ERROR, msg_prefix));
+ break;
+ }
+ }
+ else // File doesn't exist, create new file
+ {
+ if ((outfile=fopen(filepath.string().c_str(), "w"))!=NULL)
+ {
+ curl_easy_setopt(dlhandle, CURLOPT_RESUME_FROM_LARGE, 0); // start downloading from the beginning of file
+ curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, outfile);
+ }
+ else
+ {
+ msgQueue.push(Message("Failed to create " + filepath.string(), MSGTYPE_ERROR, msg_prefix));
+ break;
+ }
+ }
+
+ xferinfo.offset = iResumePosition;
+ xferinfo.timer.reset();
+ xferinfo.TimeAndSize.clear();
+ result = curl_easy_perform(dlhandle);
+ fclose(outfile);
+
+ if (result == CURLE_PARTIAL_FILE || result == CURLE_OPERATION_TIMEDOUT)
+ {
+ iRetryCount++;
+ if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath))
+ bResume = true;
+ }
+
+ } while ((result == CURLE_PARTIAL_FILE || result == CURLE_OPERATION_TIMEDOUT) && (iRetryCount <= conf.iRetries));
+
+ long int response_code = 0;
+ if (result == CURLE_HTTP_RETURNED_ERROR)
+ {
+ curl_easy_getinfo(dlhandle, CURLINFO_RESPONSE_CODE, &response_code);
+ }
+ if (result == CURLE_OK || result == CURLE_RANGE_ERROR || (result == CURLE_HTTP_RETURNED_ERROR && response_code == 416))
+ {
+ // Average download speed
+ std::ostringstream dlrate_avg;
+ std::string rate_unit;
+ progressInfo progress_info = vDownloadInfo[tid].getProgressInfo();
+ if (progress_info.rate_avg > 1048576) // 1 MB
+ {
+ progress_info.rate_avg /= 1048576;
+ rate_unit = "MB/s";
+ }
+ else
+ {
+ progress_info.rate_avg /= 1024;
+ rate_unit = "kB/s";
+ }
+ dlrate_avg << std::setprecision(2) << std::fixed << progress_info.rate_avg << rate_unit;
+
+ msgQueue.push(Message("Download complete: " + filepath.filename().string() + " (@ " + dlrate_avg.str() + ")", MSGTYPE_SUCCESS, msg_prefix));
+ }
+ else
+ {
+ msgQueue.push(Message("Download complete (" + static_cast<std::string>(curl_easy_strerror(result)) + "): " + filepath.filename().string(), MSGTYPE_WARNING, msg_prefix));
+
+ // Delete the file if download failed and was not a resume attempt or the result is zero length file
+ if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath))
+ {
+ if ((result != CURLE_PARTIAL_FILE && !bResume && result != CURLE_OPERATION_TIMEDOUT) || boost::filesystem::file_size(filepath) == 0)
+ {
+ if (!boost::filesystem::remove(filepath))
+ msgQueue.push(Message("Failed to delete " + filepath.filename().string(), MSGTYPE_ERROR, msg_prefix));
+ }
+ }
+ }
+
+ // Automatic xml creation
+ if (conf.bAutomaticXMLCreation)
+ {
+ if (result == CURLE_OK)
+ {
+ if ((gf.type & GFTYPE_EXTRA) || (conf.bRemoteXML && !bLocalXMLExists && xml.empty()))
+ createXMLQueue.push(gf);
+ }
+ }
+ }
+
+ curl_easy_cleanup(dlhandle);
+ delete api;
+
+ vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED);
+ msgQueue.push(Message("Finished all tasks", MSGTYPE_INFO, msg_prefix));
+
+ return;
+}
+
+int Downloader::progressCallbackForThread(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
+{
+ // unused so lets prevent warnings and be more pedantic
+ (void) ulnow;
+ (void) ultotal;
+
+ xferInfo* xferinfo = static_cast<xferInfo*>(clientp);
+
+ // Update progress info every 100ms
+ if (xferinfo->timer.getTimeBetweenUpdates()>=100 || dlnow == dltotal)
+ {
+ xferinfo->timer.reset();
+ progressInfo info;
+ info.dlnow = dlnow;
+ info.dltotal = dltotal;
+
+ // trying to get rate and setting to NaN if it fails
+ if (CURLE_OK != curl_easy_getinfo(xferinfo->curlhandle, CURLINFO_SPEED_DOWNLOAD, &info.rate_avg))
+ info.rate_avg = std::numeric_limits<double>::quiet_NaN();
+
+ // setting full dlwnow and dltotal
+ if (xferinfo->offset > 0)
+ {
+ info.dlnow += xferinfo->offset;
+ info.dltotal += xferinfo->offset;
+ }
+
+ // 10 second average download speed
+ // Don't use static value of 10 seconds because update interval depends on when and how often progress callback is called
+ xferinfo->TimeAndSize.push_back(std::make_pair(time(NULL), static_cast<uintmax_t>(info.dlnow)));
+ if (xferinfo->TimeAndSize.size() > 100) // 100 * 100ms = 10s
+ {
+ xferinfo->TimeAndSize.pop_front();
+ time_t time_first = xferinfo->TimeAndSize.front().first;
+ uintmax_t size_first = xferinfo->TimeAndSize.front().second;
+ time_t time_last = xferinfo->TimeAndSize.back().first;
+ uintmax_t size_last = xferinfo->TimeAndSize.back().second;
+ info.rate = (size_last - size_first) / static_cast<double>((time_last - time_first));
+ }
+ else
+ {
+ info.rate = info.rate_avg;
+ }
+
+ vDownloadInfo[xferinfo->tid].setProgressInfo(info);
+ vDownloadInfo[xferinfo->tid].setStatus(DLSTATUS_RUNNING);
+ }
+
+ return 0;
+}
+
+void Downloader::printProgress()
+{
+ // Print progress information until all threads have finished their tasks
+ ProgressBar bar(config.bUnicode, config.bColor);
+ unsigned int dl_status = DLSTATUS_NOTSTARTED;
+ while (dl_status != DLSTATUS_FINISHED)
+ {
+ dl_status = DLSTATUS_NOTSTARTED;
+
+ // Print progress information once per 100ms
+ std::this_thread::sleep_for(std::chrono::milliseconds(100));
+ std::cout << "\033[J\r" << std::flush; // Clear screen from the current line down to the bottom of the screen
+
+ // Print messages from message queue first
+ Message msg;
+ while (msgQueue.try_pop(msg))
+ {
+ std::cout << msg.getFormattedString(config.bColor, true) << std::endl;
+ if (config.bReport)
+ {
+ this->report_ofs << msg.getTimestampString() << ": " << msg.getMessage() << std::endl;
+ }
+ }
+
+ int iTermWidth = Util::getTerminalWidth();
+ double total_rate = 0;
+
+ // Create progress info text for all download threads
+ std::vector<std::string> vProgressText;
+ for (unsigned int i = 0; i < vDownloadInfo.size(); ++i)
+ {
+ std::string progress_text;
+ int bar_length = 26;
+ int min_bar_length = 5;
+
+ unsigned int status = vDownloadInfo[i].getStatus();
+ dl_status |= status;
+
+ if (status == DLSTATUS_FINISHED)
+ {
+ vProgressText.push_back("#" + std::to_string(i) + ": Finished");
+ continue;
+ }
+
+ std::string filename = vDownloadInfo[i].getFilename();
+ progressInfo progress_info = vDownloadInfo[i].getProgressInfo();
+ total_rate += progress_info.rate;
+
+ bool starting = ((0 == progress_info.dlnow) && (0 == progress_info.dltotal));
+ double fraction = starting ? 0.0 : static_cast<double>(progress_info.dlnow) / static_cast<double>(progress_info.dltotal);
+
+ char progress_percentage_text[200];
+ sprintf(progress_percentage_text, "%3.0f%% ", fraction * 100);
+ int progress_percentage_text_length = strlen(progress_percentage_text) + 1;
+
+ bptime::time_duration eta(bptime::seconds((long)((progress_info.dltotal - progress_info.dlnow) / progress_info.rate)));
+ std::stringstream eta_ss;
+ if (eta.hours() > 23)
+ {
+ eta_ss << eta.hours() / 24 << "d " <<
+ std::setfill('0') << std::setw(2) << eta.hours() % 24 << "h " <<
+ std::setfill('0') << std::setw(2) << eta.minutes() << "m " <<
+ std::setfill('0') << std::setw(2) << eta.seconds() << "s";
+ }
+ else if (eta.hours() > 0)
+ {
+ eta_ss << eta.hours() << "h " <<
+ std::setfill('0') << std::setw(2) << eta.minutes() << "m " <<
+ std::setfill('0') << std::setw(2) << eta.seconds() << "s";
+ }
+ else if (eta.minutes() > 0)
+ {
+ eta_ss << eta.minutes() << "m " <<
+ std::setfill('0') << std::setw(2) << eta.seconds() << "s";
+ }
+ else
+ {
+ eta_ss << eta.seconds() << "s";
+ }
+
+ std::string rate_unit;
+ if (progress_info.rate > 1048576) // 1 MB
+ {
+ progress_info.rate /= 1048576;
+ rate_unit = "MB/s";
+ }
+ else
+ {
+ progress_info.rate /= 1024;
+ rate_unit = "kB/s";
+ }
+
+ char progress_status_text[200]; // We're probably never going to go as high as 200 characters but it's better to use too big number here than too small
+ sprintf(progress_status_text, " %0.2f/%0.2fMB @ %0.2f%s ETA: %s", static_cast<double>(progress_info.dlnow)/1024/1024, static_cast<double>(progress_info.dltotal)/1024/1024, progress_info.rate, rate_unit.c_str(), eta_ss.str().c_str());
+ int status_text_length = strlen(progress_status_text) + 1;
+
+ if ((status_text_length + progress_percentage_text_length + bar_length) > iTermWidth)
+ bar_length -= (status_text_length + progress_percentage_text_length + bar_length) - iTermWidth;
+
+ // Don't draw progressbar if length is less than min_bar_length
+ std::string progress_bar_text;
+ if (bar_length >= min_bar_length)
+ progress_bar_text = bar.createBarString(bar_length, fraction);
+
+ progress_text = std::string(progress_percentage_text) + progress_bar_text + std::string(progress_status_text);
+ std::string filename_text = "#" + std::to_string(i) + " " + filename;
+ Util::shortenStringToTerminalWidth(filename_text);
+
+ vProgressText.push_back(filename_text);
+ vProgressText.push_back(progress_text);
+ }
+
+ // Total download speed and number of remaining tasks in download queue
+ if (dl_status != DLSTATUS_FINISHED)
+ {
+ std::ostringstream ss;
+ if (config.iThreads > 1)
+ {
+ std::string rate_unit;
+ if (total_rate > 1048576) // 1 MB
+ {
+ total_rate /= 1048576;
+ rate_unit = "MB/s";
+ }
+ else
+ {
+ total_rate /= 1024;
+ rate_unit = "kB/s";
+ }
+ ss << "Total: " << std::setprecision(2) << std::fixed << total_rate << rate_unit << " | ";
+ }
+ ss << "Remaining: " << dlQueue.size();
+ vProgressText.push_back(ss.str());
+ }
+
+ // Print progress info
+ for (unsigned int i = 0; i < vProgressText.size(); ++i)
+ {
+ std::cout << vProgressText[i] << std::endl;
+ }
+
+ // Move cursor up by vProgressText.size() rows
+ if (dl_status != DLSTATUS_FINISHED)
+ {
+ std::cout << "\033[" << vProgressText.size() << "A\r" << std::flush;
+ }
+ }
+}
+
+void Downloader::getGameDetailsThread(Config config, const unsigned int& tid)
+{
+ std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]";
+
+ API* api = new API(config.sToken, config.sSecret);
+ api->curlSetOpt(CURLOPT_SSL_VERIFYPEER, config.bVerifyPeer);
+ api->curlSetOpt(CURLOPT_CONNECTTIMEOUT, config.iTimeout);
+ if (!config.sCACertPath.empty())
+ api->curlSetOpt(CURLOPT_CAINFO, config.sCACertPath.c_str());
+
+ if (!api->init())
+ {
+ delete api;
+ msgQueue.push(Message("API init failed", MSGTYPE_ERROR, msg_prefix));
+ vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED);
+ return;
+ }
+
+ // Create new GOG website handle
+ Website* website = new Website(config);
+ if (!website->IsLoggedIn())
+ {
+ delete api;
+ delete website;
+ msgQueue.push(Message("Website not logged in", MSGTYPE_ERROR, msg_prefix));
+ vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED);
+ return;
+ }
+
+ // Set default game specific directory options to values from config
+ gameSpecificDirectoryConfig dirConfDefault;
+ dirConfDefault.sDirectory = config.sDirectory;
+ dirConfDefault.bSubDirectories = config.bSubDirectories;
+ dirConfDefault.sGameSubdir = config.sGameSubdir;
+ dirConfDefault.sInstallersSubdir = config.sInstallersSubdir;
+ dirConfDefault.sExtrasSubdir = config.sExtrasSubdir;
+ dirConfDefault.sLanguagePackSubdir = config.sLanguagePackSubdir;
+ dirConfDefault.sDLCSubdir = config.sDLCSubdir;
+ dirConfDefault.sPatchesSubdir = config.sPatchesSubdir;
+
+ gameItem game_item;
+ while (gameItemQueue.try_pop(game_item))
+ {
+ gameDetails game;
+ bool bHasDLC = !game_item.dlcnames.empty();
+
+ gameSpecificConfig conf;
+ conf.bDLC = config.bDLC;
+ conf.bIgnoreDLCCount = false;
+ conf.iInstallerLanguage = config.iInstallerLanguage;
+ conf.iInstallerPlatform = config.iInstallerPlatform;
+ conf.dirConf = dirConfDefault;
+ conf.vLanguagePriority = config.vLanguagePriority;
+ conf.vPlatformPriority = config.vPlatformPriority;
+ if (!config.bUpdateCache) // Disable game specific config files for cache update
+ {
+ int iOptionsOverridden = Util::getGameSpecificConfig(game_item.name, &conf);
+ if (iOptionsOverridden > 0)
+ {
+ std::ostringstream ss;
+ ss << game_item.name << " - " << iOptionsOverridden << " options overridden with game specific options" << std::endl;
+ if (config.bVerbose)
+ {
+ if (conf.bIgnoreDLCCount)
+ ss << "\tIgnore DLC count" << std::endl;
+ if (conf.bDLC != config.bDLC)
+ ss << "\tDLC: " << (conf.bDLC ? "true" : "false") << std::endl;
+ if (conf.iInstallerLanguage != config.iInstallerLanguage)
+ ss << "\tLanguage: " << Util::getOptionNameString(conf.iInstallerLanguage, GlobalConstants::LANGUAGES) << std::endl;
+ if (conf.vLanguagePriority != config.vLanguagePriority)
+ {
+ ss << "\tLanguage priority:" << std::endl;
+ for (unsigned int j = 0; j < conf.vLanguagePriority.size(); ++j)
+ {
+ ss << "\t " << j << ": " << Util::getOptionNameString(conf.vLanguagePriority[j], GlobalConstants::LANGUAGES) << std::endl;
+ }
+ }
+ if (conf.iInstallerPlatform != config.iInstallerPlatform)
+ ss << "\tPlatform: " << Util::getOptionNameString(conf.iInstallerPlatform, GlobalConstants::PLATFORMS) << std::endl;
+ if (conf.vPlatformPriority != config.vPlatformPriority)
+ {
+ ss << "\tPlatform priority:" << std::endl;
+ for (unsigned int j = 0; j < conf.vPlatformPriority.size(); ++j)
+ {
+ ss << "\t " << j << ": " << Util::getOptionNameString(conf.vPlatformPriority[j], GlobalConstants::PLATFORMS) << std::endl;
+ }
+ }
+ }
+ msgQueue.push(Message(ss.str(), MSGTYPE_INFO, msg_prefix));
+ }
+ }
+
+ game = api->getGameDetails(game_item.name, conf.iInstallerPlatform, conf.iInstallerLanguage, config.bDuplicateHandler);
+ if (!api->getError())
+ {
+ game.filterWithPriorities(conf);
+ Json::Value gameDetailsJSON;
+
+ if (!game_item.gamedetailsjson.empty())
+ gameDetailsJSON = game_item.gamedetailsjson;
+
+ if (game.extras.empty() && config.bExtras) // Try to get extras from account page if API didn't return any extras
+ {
+ if (gameDetailsJSON.empty())
+ gameDetailsJSON = website->getGameDetailsJSON(game_item.id);
+ game.extras = Downloader::getExtrasFromJSON(gameDetailsJSON, game_item.name, config);
+ }
+ if (config.bSaveSerials)
+ {
+ if (gameDetailsJSON.empty())
+ gameDetailsJSON = website->getGameDetailsJSON(game_item.id);
+ game.serials = Downloader::getSerialsFromJSON(gameDetailsJSON);
+ }
+ if (config.bSaveChangelogs)
+ {
+ if (gameDetailsJSON.empty())
+ gameDetailsJSON = website->getGameDetailsJSON(game_item.id);
+ game.changelog = Downloader::getChangelogFromJSON(gameDetailsJSON);
+ }
+
+ // Ignore DLC count and try to get DLCs from JSON
+ if (game.dlcs.empty() && !bHasDLC && conf.bDLC && conf.bIgnoreDLCCount)
+ {
+ if (gameDetailsJSON.empty())
+ gameDetailsJSON = website->getGameDetailsJSON(game_item.id);
+
+ game_item.dlcnames = Util::getDLCNamesFromJSON(gameDetailsJSON["dlcs"]);
+ bHasDLC = !game_item.dlcnames.empty();
+ }
+
+ if (game.dlcs.empty() && bHasDLC && conf.bDLC)
+ {
+ for (unsigned int j = 0; j < game_item.dlcnames.size(); ++j)
+ {
+ gameDetails dlc;
+ dlc = api->getGameDetails(game_item.dlcnames[j], conf.iInstallerPlatform, conf.iInstallerLanguage, config.bDuplicateHandler);
+ dlc.filterWithPriorities(conf);
+ if (dlc.extras.empty() && config.bExtras) // Try to get extras from account page if API didn't return any extras
+ {
+ if (gameDetailsJSON.empty())
+ gameDetailsJSON = website->getGameDetailsJSON(game_item.id);
+
+ // Make sure we get extras for the right DLC
+ for (unsigned int k = 0; k < gameDetailsJSON["dlcs"].size(); ++k)
+ {
+ std::vector<std::string> urls;
+ if (gameDetailsJSON["dlcs"][k].isMember("extras"))
+ Util::getDownloaderUrlsFromJSON(gameDetailsJSON["dlcs"][k]["extras"], urls);
+
+ if (!urls.empty())
+ {
+ if (urls[0].find("/" + game_item.dlcnames[j] + "/") != std::string::npos)
+ {
+ dlc.extras = Downloader::getExtrasFromJSON(gameDetailsJSON["dlcs"][k], game_item.dlcnames[j], config);
+ }
+ }
+ }
+ }
+
+ if (config.bSaveSerials)
+ {
+ if (gameDetailsJSON.empty())
+ gameDetailsJSON = website->getGameDetailsJSON(game_item.id);
+
+ // Make sure we save serial for the right DLC
+ for (unsigned int k = 0; k < gameDetailsJSON["dlcs"].size(); ++k)
+ {
+ std::vector<std::string> urls;
+ if (gameDetailsJSON["dlcs"][k].isMember("cdKey") && gameDetailsJSON["dlcs"][k].isMember("downloads"))
+ {
+ // Assuming that only DLC with installers can have serial
+ Util::getDownloaderUrlsFromJSON(gameDetailsJSON["dlcs"][k]["downloads"], urls);
+ }
+
+ if (!urls.empty())
+ {
+ if (urls[0].find("/" + game_item.dlcnames[j] + "/") != std::string::npos)
+ {
+ dlc.serials = Downloader::getSerialsFromJSON(gameDetailsJSON["dlcs"][k]);
+ }
+ }
+ }
+ }
+
+ if (config.bSaveChangelogs)
+ {
+ if (gameDetailsJSON.empty())
+ gameDetailsJSON = website->getGameDetailsJSON(game_item.id);
+
+ // Make sure we save changelog for the right DLC
+ for (unsigned int k = 0; k < gameDetailsJSON["dlcs"].size(); ++k)
+ {
+ std::vector<std::string> urls;
+ if (gameDetailsJSON["dlcs"][k].isMember("changelog") && gameDetailsJSON["dlcs"][k].isMember("downloads"))
+ {
+ // Assuming that only DLC with installers can have changelog
+ Util::getDownloaderUrlsFromJSON(gameDetailsJSON["dlcs"][k]["downloads"], urls);
+ }
+
+ if (!urls.empty())
+ {
+ if (urls[0].find("/" + game_item.dlcnames[j] + "/") != std::string::npos)
+ {
+ dlc.changelog = Downloader::getChangelogFromJSON(gameDetailsJSON["dlcs"][k]);
+ }
+ }
+ }
+ }
+
+ game.dlcs.push_back(dlc);
+ }
+ }
+
+ game.makeFilepaths(conf.dirConf);
+
+ if (!config.bUpdateCheck)
+ gameDetailsQueue.push(game);
+ else
+ { // Update check, only add games that have updated files
+ for (unsigned int j = 0; j < game.installers.size(); ++j)
+ {
+ if (game.installers[j].updated)
+ {
+ gameDetailsQueue.push(game);
+ break; // add the game only once
+ }
+ }
+ }
+ }
+ else
+ {
+ msgQueue.push(Message(api->getErrorMessage(), MSGTYPE_ERROR, msg_prefix));
+ api->clearError();
+ continue;
+ }
+ }
+ vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED);
+ delete api;
+ delete website;
+ return;
+}
diff --git a/src/gamedetails.cpp b/src/gamedetails.cpp
index c3d6a5b..d3c7530 100644
--- a/src/gamedetails.cpp
+++ b/src/gamedetails.cpp
@@ -132,6 +132,13 @@ void gameDetails::makeFilepaths(const gameSpecificDirectoryConfig& config)
filepath = Util::makeFilepath(directory, this->dlcs[i].extras[j].path, this->gamename, subdir, 0, this->dlcs[i].gamename);
this->dlcs[i].extras[j].setFilepath(filepath);
}
+
+ for (unsigned int j = 0; j < this->dlcs[i].languagepacks.size(); ++j)
+ {
+ subdir = config.bSubDirectories ? config.sDLCSubdir + "/" + config.sLanguagePackSubdir : "";
+ filepath = Util::makeFilepath(directory, this->dlcs[i].languagepacks[j].path, this->gamename, subdir, 0, this->dlcs[i].gamename);
+ this->dlcs[i].languagepacks[j].setFilepath(filepath);
+ }
}
}
diff --git a/src/util.cpp b/src/util.cpp
index 3d9dfed..f88f26f 100644
--- a/src/util.cpp
+++ b/src/util.cpp
@@ -8,7 +8,7 @@
#include <boost/filesystem.hpp>
#include <boost/algorithm/string/case_conv.hpp>
-#include <tinyxml.h>
+#include <tinyxml2.h>
#include <json/json.h>
#include <fstream>
#include <sys/ioctl.h>
@@ -132,11 +132,11 @@ int Util::createXML(std::string filepath, uintmax_t chunk_size, std::string xml_
<< "Chunks: " << chunks << std::endl
<< "Chunk size: " << (chunk_size >> 20) << " MB" << std::endl;
- TiXmlDocument xml;
- TiXmlElement *fileElem = new TiXmlElement("file");
- fileElem->SetAttribute("name", filename);
+ tinyxml2::XMLDocument xml;
+ tinyxml2::XMLElement *fileElem = xml.NewElement("file");
+ fileElem->SetAttribute("name", filename.c_str());
fileElem->SetAttribute("chunks", chunks);
- fileElem->SetAttribute("total_size", std::to_string(filesize));
+ fileElem->SetAttribute("total_size", std::to_string(filesize).c_str());
std::cout << "Getting MD5 for chunks" << std::endl;
@@ -175,12 +175,12 @@ int Util::createXML(std::string filepath, uintmax_t chunk_size, std::string xml_
free(chunk);
- TiXmlElement *chunkElem = new TiXmlElement("chunk");
+ tinyxml2::XMLElement *chunkElem = xml.NewElement("chunk");
chunkElem->SetAttribute("id", i);
- chunkElem->SetAttribute("from", std::to_string(range_begin));
- chunkElem->SetAttribute("to", std::to_string(range_end));
+ chunkElem->SetAttribute("from", std::to_string(range_begin).c_str());
+ chunkElem->SetAttribute("to", std::to_string(range_end).c_str());
chunkElem->SetAttribute("method", "md5");
- TiXmlText *text = new TiXmlText(hash);
+ tinyxml2::XMLText *text = xml.NewText(hash.c_str());
chunkElem->LinkEndChild(text);
fileElem->LinkEndChild(chunkElem);
@@ -192,15 +192,15 @@ int Util::createXML(std::string filepath, uintmax_t chunk_size, std::string xml_
rhash_print(rhash_result, rhash_context, RHASH_MD5, RHPR_HEX);
rhash_free(rhash_context);
- std::string file_md5 = rhash_result;
- std::cout << std::endl << "MD5: " << file_md5 << std::endl;
- fileElem->SetAttribute("md5", file_md5);
+ std::cout << std::endl << "MD5: " << rhash_result << std::endl;
+ fileElem->SetAttribute("md5", rhash_result);
xml.LinkEndChild(fileElem);
std::cout << "Writing XML: " << filenameXML << std::endl;
if ((xmlfile=fopen(filenameXML.c_str(), "w"))!=NULL) {
- xml.Print(xmlfile);
+ tinyxml2::XMLPrinter printer(xmlfile);
+ xml.Print(&printer);
fclose(xmlfile);
res = 1;
} else {
@@ -540,3 +540,46 @@ void Util::parseOptionString(const std::string &option_string, std::vector<unsig
type |= value;
}
}
+
+std::string Util::getLocalFileHash(const std::string& xml_dir, const std::string& filepath, const std::string& gamename)
+{
+ std::string localHash;
+ boost::filesystem::path path = filepath;
+ boost::filesystem::path local_xml_file;
+ if (!gamename.empty())
+ local_xml_file = xml_dir + "/" + gamename + "/" + path.filename().string() + ".xml";
+ else
+ local_xml_file = xml_dir + "/" + path.filename().string() + ".xml";
+
+ if (boost::filesystem::exists(local_xml_file))
+ {
+ tinyxml2::XMLDocument local_xml;
+ local_xml.LoadFile(local_xml_file.string().c_str());
+ tinyxml2::XMLElement *fileElem = local_xml.FirstChildElement("file");
+
+ if (fileElem)
+ {
+ localHash = fileElem->Attribute("md5");
+ }
+ }
+ else if (boost::filesystem::exists(path) && boost::filesystem::is_regular_file(path))
+ {
+ localHash = Util::getFileHash(path.string(), RHASH_MD5);
+ }
+
+ return localHash;
+}
+
+void Util::shortenStringToTerminalWidth(std::string& str)
+{
+ int iStrLen = static_cast<int>(str.length());
+ int iTermWidth = Util::getTerminalWidth();
+ if (iStrLen >= iTermWidth)
+ {
+ size_t chars_to_remove = (iStrLen - iTermWidth) + 4;
+ size_t middle = iStrLen / 2;
+ size_t pos1 = middle - (chars_to_remove / 2);
+ size_t pos2 = middle + (chars_to_remove / 2);
+ str.replace(str.begin()+pos1, str.begin()+pos2, "...");
+ }
+}
diff --git a/src/website.cpp b/src/website.cpp
index 2e68bed..0769968 100644
--- a/src/website.cpp
+++ b/src/website.cpp
@@ -31,6 +31,8 @@ Website::Website(Config &conf)
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_TIME, 30);
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_LIMIT, 200);
+ if (!config.sCACertPath.empty())
+ curl_easy_setopt(curlhandle, CURLOPT_CAINFO, config.sCACertPath.c_str());
}
Website::~Website()
@@ -80,6 +82,11 @@ std::string Website::getResponse(const std::string& url)
else
std::cout << "failed to get error code: " << curl_easy_strerror(result) << " (" << url << ")" << std::endl;
}
+ else if (result == CURLE_SSL_CACERT)
+ {
+ std::cout << "Try using CA certificate bundle from cURL: https://curl.haxx.se/ca/cacert.pem" << std::endl;
+ std::cout << "Use --cacert to set the path for CA certificate bundle" << std::endl;
+ }
}
return response;
@@ -150,6 +157,7 @@ std::vector<gameItem> Website::getGames()
{
for (unsigned int i = 0; i < root["products"].size(); ++i)
{
+ std::cerr << "\033[KGetting game names " << "(" << root["page"].asInt() << "/" << root["totalPages"].asInt() << ") " << i+1 << " / " << root["products"].size() << "\r" << std::flush;
Json::Value product = root["products"][i];
gameItem game;
game.name = product["slug"].asString();
@@ -196,14 +204,19 @@ std::vector<gameItem> Website::getGames()
}
}
+ if (!bDownloadDLCInfo && !config.gamehasdlc.empty())
+ {
+ if (config.gamehasdlc.isBlacklisted(game.name))
+ bDownloadDLCInfo = true;
+ }
+
// Check game specific config
if (!config.bUpdateCache) // Disable game specific config files for cache update
{
gameSpecificConfig conf;
- conf.bIgnoreDLCCount = false; // Assume false
+ conf.bIgnoreDLCCount = bDownloadDLCInfo;
Util::getGameSpecificConfig(game.name, &conf);
- if (conf.bIgnoreDLCCount)
- bDownloadDLCInfo = true;
+ bDownloadDLCInfo = conf.bIgnoreDLCCount;
}
if (bDownloadDLCInfo && !config.sGameRegex.empty())
@@ -229,6 +242,7 @@ std::vector<gameItem> Website::getGames()
}
i++;
} while (!bAllPagesParsed);
+ std::cerr << std::endl;
delete jsonparser;
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/lgogdownloader.git
More information about the Pkg-games-commits
mailing list