From a51c8117d0c5cfe3ad4087577305a7621931afe6 Mon Sep 17 00:00:00 2001 From: katboi01 Date: Sun, 16 Mar 2025 20:41:53 +0100 Subject: [PATCH] undo downloader changes --- modules/KF3/downloader.py | 193 ++++++++++---------------------------- 1 file changed, 47 insertions(+), 146 deletions(-) diff --git a/modules/KF3/downloader.py b/modules/KF3/downloader.py index a2b3b9f..1951f65 100644 --- a/modules/KF3/downloader.py +++ b/modules/KF3/downloader.py @@ -9,7 +9,6 @@ import platform import aiohttp from UnityPy import enums -import requests from ..Shared.utility import divide_chunks from ..Shared.downloading import download_bytes, download_text @@ -42,7 +41,7 @@ def encode(text): text[i] = text[i] ^ key[i % (len(key))] return hashed[:4].hex() + text.hex() + checksum.hex() -async def download_cache(server_name, server_app : str, server_cdn : str = None): +async def download_cache(server_name, server : str): session = aiohttp.ClientSession() downloaded_files = [] @@ -61,7 +60,7 @@ async def download_cache(server_name, server_app : str, server_cdn : str = None) old_mst_ver = dict([(entry["type"], entry["version"]) for entry in json.load(file)]) param = encode(json.dumps({'dmm_viewer_id':0})) - request = await download_bytes(server_app + "paradesv/common/MstVersion.do?param=" + param, session) + request = await download_bytes(server + "paradesv/common/MstVersion.do?param=" + param, session) result = gzip.decompress(request) new_mst = json.loads(result) new_mst_ver = dict([(entry["type"], entry["version"]) for entry in new_mst["mst_ver"]]) @@ -75,7 +74,7 @@ async def download_cache(server_name, server_app : str, server_cdn : str = None) downloaded_files.append(key) param = encode(json.dumps({'type':key, 'dmm_viewer_id':0})) - request = await download_bytes(server_app + "paradesv/common/MstData.do?param=" + param, session) + request = await download_bytes(server + "paradesv/common/MstData.do?param=" + param, session) result = gzip.decompress(request) response = json.loads(result) data = base64.b64decode(response["data"]) @@ -84,14 +83,8 @@ async def download_cache(server_name, server_app : str, server_cdn : str = None) with open(file_path_json, "wt", encoding="utf-8") as out_file: data = gzip.decompress(data) data = json.loads(data) - if key == "GACHA_DATA": - await download_banners(data, server_name, server_app, server_cdn, session) - if key == "BANNER_DATA": - await download_banner_data(data, server_name, server_cdn, session) - if key == "EVENT_BANNER_DATA": - await download_event_banner_data(data, server_name, server_cdn, session) - if key == "EVENT_DATA": - await download_event_data(data, server_name, server_cdn, session) + # if key == "GACHA_DATA": + # download_banners(data, server_name) json.dump(data, out_file, ensure_ascii=False, indent=1) old_mst_ver[key] = new_mst_ver[key] @@ -103,21 +96,21 @@ async def download_cache(server_name, server_app : str, server_cdn : str = None) await session.close() return downloaded_files -async def download_banners(gacha_data, server_name, server_app, server_cdn, session): - path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\banners\\" +async def download_banners(gacha_data, server_name, session): + path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/banners/" os.makedirs(path, exist_ok=True) - async def download_banner(entry): + for entry in gacha_data: banner_name = entry["banner"] if banner_name == "" or banner_name == None: - return + continue banner_name += ".png" file_path = path + banner_name if os.path.exists(file_path): - return + continue - file_url_alt = f"{server_cdn}Texture2D/GachaTop/{banner_name}" - file_url = f"{server_app}Texture2D/GachaTop/{banner_name}" + file_url = "https://parade-mobile-prod-cdn.kemono-friends-3.jp/Texture2D/GachaTop/" + banner_name + file_url_alt = "https://parade-mobile-develop01-app.kemono-friends-3.jp/Texture2D/GachaTop/" + banner_name status = 0 async with session.get(file_url) as resp: @@ -127,118 +120,11 @@ async def download_banners(gacha_data, server_name, server_app, server_cdn, sess async with session.get(file_url_alt) as resp: response = await resp.read() status = resp.status - if status != 200: return + if status != 200: continue with open(file_path, "wb") as file: file.write(response) - - chunked_files_to_download = list(divide_chunks(gacha_data, 20)) - - for chunk in chunked_files_to_download: - tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk] - await asyncio.wait(tasks) - -async def download_banner_data(gacha_data, server_name, server_cdn, session : aiohttp.ClientSession): - path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\" - - async def download_banner(entry): - name = entry["bannerName"] - if name == "" or name == None: - return - bannerImagePath = "Texture2D/HomeBanner/home_banner_" + name + ".png" - bannerImagePathByQuestTop = "Texture2D/QuestTop/questtop_banner_" + name + ".png" - bannerImagePathEvent = "Texture2D/EventTop/eventtop_banner_" + name + ".png" - banners = [bannerImagePath, bannerImagePathByQuestTop, bannerImagePathEvent] - - for banner in banners: - file_path = path + banner - if os.path.exists(file_path): - continue - - fileDir = "/".join(file_path.split("/")[:-1]) - os.makedirs(fileDir, exist_ok=True) - file_url = f"{server_cdn}{banner}" - - try: - with open(file_path, 'wb') as out_file: - req = requests.get(file_url, stream=True) - if req.status_code != 200: raise Exception("lol") - out_file.write(req.content) - except Exception as ex: - os.remove(file_path) - print(ex) - continue - - chunked_files_to_download = list(divide_chunks(gacha_data, 20)) - - for chunk in chunked_files_to_download: - tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk] - await asyncio.wait(tasks) - -async def download_event_banner_data(gacha_data, server_name, server_cdn, session : aiohttp.ClientSession): - path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\" - - async def download_banner(entry): - name = entry["bannerName"] - if name == "" or name == None: - return - banner = "Texture2D/HomeBigBanner/home_bigbanner_" + name + ".png" - - file_path = path + banner - if os.path.exists(file_path): - return - fileDir = "/".join(file_path.split("/")[:-1]) - os.makedirs(fileDir, exist_ok=True) - file_url = f"{server_cdn}{banner}" - - try: - with open(file_path, 'wb') as out_file: - req = requests.get(file_url, stream=True) - if req.status_code != 200: raise Exception("lol") - out_file.write(req.content) - except Exception as ex: - os.remove(file_path) - print(ex) - - chunked_files_to_download = list(divide_chunks(gacha_data, 20)) - - for chunk in chunked_files_to_download: - tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk] - await asyncio.wait(tasks) - - -async def download_event_data(gacha_data, server_name, server_cdn, session : aiohttp.ClientSession): - path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\" - - async def download_banner(entry): - name = entry["missionBannerFilename"] - if name == "" or name == None: - return - banner = "Texture2D/Mission/" + name + ".png" - - file_path = path + banner - if os.path.exists(file_path): - return - - fileDir = "/".join(file_path.split("/")[:-1]) - os.makedirs(fileDir, exist_ok=True) - file_url = f"{server_cdn}{banner}" - - try: - with open(file_path, 'wb') as out_file: - req = requests.get(file_url, stream=True) - if req.status_code != 200: raise Exception("lol") - out_file.write(req.content) - except Exception as ex: - os.remove(file_path) - print(ex) - - chunked_files_to_download = list(divide_chunks(gacha_data, 20)) - - for chunk in chunked_files_to_download: - tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk] - await asyncio.wait(tasks) async def download_files(server_name, asset_bundle_url, srv_platform : str): def parse_ab_list(filecontent : str): @@ -267,11 +153,7 @@ async def download_files(server_name, asset_bundle_url, srv_platform : str): extract(data, extract_path) session = aiohttp.ClientSession() - - if platform.system() == "Windows": - path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\assets\\{srv_platform}\\" - else: - path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/assets/{srv_platform}/" + path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/assets/{srv_platform}/" os.makedirs(path, exist_ok=True) @@ -308,7 +190,7 @@ async def download_files(server_name, asset_bundle_url, srv_platform : str): files_to_download.append(key) - chunked_files_to_download = list(divide_chunks(files_to_download, 10)) + chunked_files_to_download = list(divide_chunks(files_to_download, 5)) for chunk in chunked_files_to_download: tasks = [asyncio.create_task(download_file(url_assets, file, path, session)) for file in chunk] @@ -345,30 +227,49 @@ async def convert_files(): print("Conversion failed", f) async def manual(): + + # session = aiohttp.ClientSession() + + # path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/cache/" + + # await session.close() + + # return + downloaded_cache = {} downloaded_files = {} - - server_cdn = "https://kf3-prod-cdn.kf3.wayi.com.tw/" - server_app = "https://kf3-prod-app.kf3.wayi.com.tw/" async with aiohttp.ClientSession() as session: param = encode(json.dumps({"version":"1.0.0","dmm_viewer_id":0,"platform":1})) - request = await download_bytes(server_app + "paradesv/common/GetUrl.do?param=" + param, session) + request = await download_bytes(servers[0] + "paradesv/common/GetUrl.do?param=" + param, session) result = gzip.decompress(request) response = json.loads(result) asset_bundle_url = response["asset_bundle_url"] urlName = asset_bundle_url.split("-")[2] - print("downloading from", server_app) - downloaded_cache = await download_cache("TW", server_app, server_cdn) - downloaded_files = await download_files("TW", asset_bundle_url, "Android") + print("downloading from", servers[0]) + downloaded_cache = await download_cache(urlName, servers[0]) + downloaded_files = await download_files(urlName, asset_bundle_url, "Windows") - # if downloaded_cache != [] and downloaded_cache != None: - # with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_cache.json", "wt", encoding="utf-8") as file: - # json.dump(downloaded_cache, file, ensure_ascii=False, indent=1) - # if downloaded_files != [] and downloaded_files != None: - # with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_files.json", "wt", encoding="utf-8") as file: - # json.dump(downloaded_files, file, ensure_ascii=False, indent=1) + if downloaded_cache != [] and downloaded_cache != None: + with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_cache.json", "wt", encoding="utf-8") as file: + json.dump(downloaded_cache, file, ensure_ascii=False, indent=1) + if downloaded_files != [] and downloaded_files != None: + with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_files.json", "wt", encoding="utf-8") as file: + json.dump(downloaded_files, file, ensure_ascii=False, indent=1) + + print("downloading from", servers[1]) + asset_bundle_url = "https://parade-mobile-develop01-app.kemono-friends-3.jp/AssetBundles/0.0.0/latest" + urlName = asset_bundle_url.split("-")[2] + downloaded_cache = await download_cache(urlName, servers[1]) + downloaded_files = await download_files(urlName, asset_bundle_url, "Windows") + + if downloaded_cache != [] and downloaded_cache != None: + with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_dev_cache.json", "wt", encoding="utf-8") as file: + json.dump(downloaded_cache, file, ensure_ascii=False, indent=1) + if downloaded_files != [] and downloaded_files != None: + with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_dev_files.json", "wt", encoding="utf-8") as file: + json.dump(downloaded_files, file, ensure_ascii=False, indent=1) if __name__ == "__main__": asyncio.run(manual())