diff --git a/.gitmodules b/.gitmodules index 7833dea..14b317e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "request_wrapper"] path = request_wrapper url = https://gitlab.com/snippets/1900824.git +[submodule "handling-http-429-with-tenacity"] + path = handling-http-429-with-tenacity + url = https://github.com/alexwlchan/handling-http-429-with-tenacity.git diff --git a/handling-http-429-with-tenacity b/handling-http-429-with-tenacity new file mode 160000 index 0000000..6dd6bb5 --- /dev/null +++ b/handling-http-429-with-tenacity @@ -0,0 +1 @@ +Subproject commit 6dd6bb517515b37e0d0e89aa07767b32072c5ee8 diff --git a/mcUp.py b/mcUp.py index c20e967..7114e1a 100755 --- a/mcUp.py +++ b/mcUp.py @@ -10,6 +10,7 @@ import argparse import shutil import hashlib from os import error +from urllib.parse import unquote parser = argparse.ArgumentParser(description='A command-line tool to update a Minecraft Server.') parser.add_argument('api', metavar='api', help='which API to use') parser.add_argument('project', metavar='project', help='which project to query for') diff --git a/parsers/modrinth.py b/parsers/modrinth.py index 5f9b2b5..35c8dd9 100644 --- a/parsers/modrinth.py +++ b/parsers/modrinth.py @@ -8,6 +8,7 @@ from request_wrapper import requests_wrapper as requests import datetime import iso8601 import pytz +from urllib.parse import unquote debug = True if debug == True: import logging @@ -18,6 +19,8 @@ if debug == True: requests_log.setLevel(logging.DEBUG) requests_log.propagate = True timeoutTime = 1 +# Setup session, this lets the parser re-use the connection instead of establishing a new connection for EACH request, not only does this cause a HUGE performance boost, it's also nicer to the API. +session = requests.Session() base_api_url = "https://api.modrinth.com:443/api/v1" def failCheck(response, functOrigin): print("Status Code is: "+str(response.status_code)) @@ -40,9 +43,9 @@ def modInfo(project): if "modInfo" in dataCache: print("Returning cached data!") return dataCache["modInfo"] - response = requests.get(base_api_url+"/mod/"+project, family=socket.AF_INET) + response = session.get(base_api_url+"/mod/"+project, family=socket.AF_INET) if failCheck(response, "modInfo") == True: #Attempt to requery API - response = requests.get(base_api_url+"/mod/"+project, family=socket.AF_INET) + response = session.get(base_api_url+"/mod/"+project, family=socket.AF_INET) api_response = response.json() cacheData("modInfo", api_response) return api_response @@ -66,9 +69,9 @@ def getAllModVersionInfo(project): responseList = [] numberOfVersions = len(versions) for item in range(numberOfVersions): - response = requests.get(base_api_url+"/version/"+versions[item], family=socket.AF_INET) + response = session.get(base_api_url+"/version/"+versions[item], family=socket.AF_INET) if failCheck(response, "getAllModVersionInfo") == True: #Attempt to requery API - response = requests.get(base_api_url+"/version/"+versions[item], family=socket.AF_INET) + response = session.get(base_api_url+"/version/"+versions[item], family=socket.AF_INET) api_response = response.json() responseList.append(api_response) cacheData("getAllMinecraftVersionInfo", responseList) @@ -122,7 +125,7 @@ def getDownloadURL(project, versionID): workingDict2 = workingList[0] workingDict3 = workingDict2["hashes"] #print(workingDict3) - downloadURLs[versions[item]] = workingDict2["url"] + downloadURLs[versions[item]] = unquote(workingDict2["url"]) downloadSHA1[versions[item]] = workingDict3["sha1"] downloadFilenames[versions[item]] = workingDict2["filename"] #print(downloadURLs)