import base64 import os import gzip import json import asyncio import hashlib import platform import aiohttp from UnityPy import enums import requests from ..Shared.utility import divide_chunks from ..Shared.downloading import download_bytes, download_text from ..Shared.convert import convert, extract servers = [ "https://parade-mobile-stg-app.kemono-friends-3.jp/", "https://parade-mobile-develop01-app.kemono-friends-3.jp/", #"https://parade-mobile-develop02-app.kemono-friends-3.jp/paradesv/", #"https://parade-mobile-develop03-app.kemono-friends-3.jp/paradesv/", #"https://parade-mobile-develop04-app.kemono-friends-3.jp/paradesv/", ] def decode(data): # cut off the md5 checksum at the end and the four bytes at the start hash = bytearray.fromhex(data[:-(2*16)][2*4:]) key = hashlib.md5(bytearray.fromhex(data[:2*4])).digest() # md5 the key # xor with the key for i in range(0, len(hash)): hash[i] = hash[i] ^ key[i % (len(key))] return hash.decode("utf-8") def encode(text): hashed = hashlib.md5(text.encode()).digest() checksum = hashlib.md5((text + "DARAPAB ").encode()).digest() key = hashlib.md5(hashed[:4]).digest() # md5 the key # xor the data with the key text = bytearray(text.encode()) for i in range(0, len(text)): text[i] = text[i] ^ key[i % (len(key))] return hashed[:4].hex() + text.hex() + checksum.hex() async def download_cache(server_name, server_app : str, server_cdn : str = None): session = aiohttp.ClientSession() downloaded_files = [] if platform.system() == "Windows": path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\cache\\" else: path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/cache/" os.makedirs(path, exist_ok=True) new_mst_ver = {} old_mst_ver = {} file_path_mst = path + "mstVersion.txt" if os.path.exists(file_path_mst): with open(file_path_mst, "rt", encoding="utf-8") as file: old_mst_ver = dict([(entry["type"], entry["version"]) for entry in json.load(file)]) param = encode(json.dumps({'dmm_viewer_id':0})) request = await download_bytes(server_app + "paradesv/common/MstVersion.do?param=" + param, session) result = gzip.decompress(request) new_mst = json.loads(result) new_mst_ver = dict([(entry["type"], entry["version"]) for entry in new_mst["mst_ver"]]) for key in new_mst_ver: file_path_gzip = path + key + ".d" file_path_json = path + key + ".json" if os.path.exists(file_path_gzip) and os.path.exists(file_path_json) and key in old_mst_ver: if new_mst_ver[key] == old_mst_ver[key]: continue downloaded_files.append(key) param = encode(json.dumps({'type':key, 'dmm_viewer_id':0})) request = await download_bytes(server_app + "paradesv/common/MstData.do?param=" + param, session) result = gzip.decompress(request) response = json.loads(result) data = base64.b64decode(response["data"]) with open(file_path_gzip, "wb") as file: file.write(data) with open(file_path_json, "wt", encoding="utf-8") as out_file: data = gzip.decompress(data) data = json.loads(data) if key == "GACHA_DATA": await download_banners(data, server_name, server_app, server_cdn, session) if key == "BANNER_DATA": await download_banner_data(data, server_name, server_cdn, session) if key == "EVENT_BANNER_DATA": await download_event_banner_data(data, server_name, server_cdn, session) if key == "EVENT_DATA": await download_event_data(data, server_name, server_cdn, session) json.dump(data, out_file, ensure_ascii=False, indent=1) old_mst_ver[key] = new_mst_ver[key] with open(file_path_mst, "wt", encoding="utf-8") as file: new_json = [{"type":type, "version":version} for type, version in zip(old_mst_ver.keys(), old_mst_ver.values())] json.dump(new_json, file, ensure_ascii=False) await session.close() return downloaded_files async def download_banners(gacha_data, server_name, server_app, server_cdn, session): path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\banners\\" os.makedirs(path, exist_ok=True) async def download_banner(entry): banner_name = entry["banner"] if banner_name == "" or banner_name == None: return banner_name += ".png" file_path = path + banner_name if os.path.exists(file_path): return file_url_alt = f"{server_cdn}Texture2D/GachaTop/{banner_name}" file_url = f"{server_app}Texture2D/GachaTop/{banner_name}" status = 0 async with session.get(file_url) as resp: response = await resp.read() status = resp.status if status != 200: async with session.get(file_url_alt) as resp: response = await resp.read() status = resp.status if status != 200: return with open(file_path, "wb") as file: file.write(response) chunked_files_to_download = list(divide_chunks(gacha_data, 20)) for chunk in chunked_files_to_download: tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk] await asyncio.wait(tasks) async def download_banner_data(gacha_data, server_name, server_cdn, session : aiohttp.ClientSession): path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\" async def download_banner(entry): name = entry["bannerName"] if name == "" or name == None: return bannerImagePath = "Texture2D/HomeBanner/home_banner_" + name + ".png" bannerImagePathByQuestTop = "Texture2D/QuestTop/questtop_banner_" + name + ".png" bannerImagePathEvent = "Texture2D/EventTop/eventtop_banner_" + name + ".png" banners = [bannerImagePath, bannerImagePathByQuestTop, bannerImagePathEvent] for banner in banners: file_path = path + banner if os.path.exists(file_path): continue fileDir = "/".join(file_path.split("/")[:-1]) os.makedirs(fileDir, exist_ok=True) file_url = f"{server_cdn}{banner}" try: with open(file_path, 'wb') as out_file: req = requests.get(file_url, stream=True) if req.status_code != 200: raise Exception("lol") out_file.write(req.content) except Exception as ex: os.remove(file_path) print(ex) continue chunked_files_to_download = list(divide_chunks(gacha_data, 20)) for chunk in chunked_files_to_download: tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk] await asyncio.wait(tasks) async def download_event_banner_data(gacha_data, server_name, server_cdn, session : aiohttp.ClientSession): path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\" async def download_banner(entry): name = entry["bannerName"] if name == "" or name == None: return banner = "Texture2D/HomeBigBanner/home_bigbanner_" + name + ".png" file_path = path + banner if os.path.exists(file_path): return fileDir = "/".join(file_path.split("/")[:-1]) os.makedirs(fileDir, exist_ok=True) file_url = f"{server_cdn}{banner}" try: with open(file_path, 'wb') as out_file: req = requests.get(file_url, stream=True) if req.status_code != 200: raise Exception("lol") out_file.write(req.content) except Exception as ex: os.remove(file_path) print(ex) chunked_files_to_download = list(divide_chunks(gacha_data, 20)) for chunk in chunked_files_to_download: tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk] await asyncio.wait(tasks) async def download_event_data(gacha_data, server_name, server_cdn, session : aiohttp.ClientSession): path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\" async def download_banner(entry): name = entry["missionBannerFilename"] if name == "" or name == None: return banner = "Texture2D/Mission/" + name + ".png" file_path = path + banner if os.path.exists(file_path): return fileDir = "/".join(file_path.split("/")[:-1]) os.makedirs(fileDir, exist_ok=True) file_url = f"{server_cdn}{banner}" try: with open(file_path, 'wb') as out_file: req = requests.get(file_url, stream=True) if req.status_code != 200: raise Exception("lol") out_file.write(req.content) except Exception as ex: os.remove(file_path) print(ex) chunked_files_to_download = list(divide_chunks(gacha_data, 20)) for chunk in chunked_files_to_download: tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk] await asyncio.wait(tasks) async def download_files(server_name, asset_bundle_url, srv_platform : str): def parse_ab_list(filecontent : str): out = {} lines = filecontent.replace("\r\n", "\n").split('\n') for line in lines: split = line.split('\t') if len(split) > 1: out[split[0]] = split[-2] return out async def download_file(url_assets : str, file_name : str, download_path : str, session : aiohttp.ClientSession): data = await download_bytes(url_assets + file_name, session) if data != None: with open(download_path + file_name, "wb") as file: file.write(data) if server_name == "develop01": convert_path = f"/var/www/html/Katworks/KF/assets/KF3/WebGL/assets/" + file_name extract_path = f"/var/www/html/Katworks/KF/assets/KF3/extracted/" try: convert(data, convert_path, enums.BuildTarget.WebGL) except: with open(convert_path, "wb") as file: file.write(data) if file_name.endswith(".png"): extract(data, extract_path) session = aiohttp.ClientSession() if platform.system() == "Windows": path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\assets\\{srv_platform}\\" else: path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/assets/{srv_platform}/" os.makedirs(path, exist_ok=True) files_to_download = [] url_base = asset_bundle_url + "/" + srv_platform + "/1.0.0/ja/" url_list = url_base + "ab_list.txt" url_env = url_base + "ab_env.txt" url_assets = url_base + "assets/" file_path_env = path + "ab_env.txt" file_path_list = path + "ab_list.txt" old_ab_env = "" if os.path.exists(file_path_env): with open(file_path_env, "rt") as file: old_ab_env = file.read() new_ab_env = await download_text(url_env, session) if new_ab_env != old_ab_env: old_ab_list = {} if os.path.exists(file_path_list): with open(file_path_list, "rt") as file: old_ab_list = parse_ab_list(file.read()) new_ab_list_file = await download_text(url_list, session) new_ab_list = parse_ab_list(new_ab_list_file) for key in new_ab_list: file_path = path + key if os.path.exists(file_path) and key in old_ab_list: if new_ab_list[key] == old_ab_list[key]: continue files_to_download.append(key) chunked_files_to_download = list(divide_chunks(files_to_download, 10)) for chunk in chunked_files_to_download: tasks = [asyncio.create_task(download_file(url_assets, file, path, session)) for file in chunk] await asyncio.wait(tasks) print(chunk) print() with open(file_path_env, "wt", encoding="utf-8") as file: file.write(new_ab_env) with open(file_path_list, "wt", encoding="utf-8") as file: file.write(new_ab_list_file) await session.close() return files_to_download async def convert_files(): directory = f"/var/www/html/Katworks/KF/assets/KF3/develop01/assets/Windows/" with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_dev_files.json", "rt", encoding="utf-8") as file: files_to_convert = json.load(file) for file_name in os.listdir(directory): if file_name not in files_to_convert: continue f = os.path.join(directory, file_name) if not os.path.isfile(f): return convert_path = f"/var/www/html/Katworks/KF/assets/KF3/WebGL/assets/" + file_name try: print(f) convert(f, convert_path, enums.BuildTarget.WebGL) except: print("Conversion failed", f) async def manual(): downloaded_cache = {} downloaded_files = {} server_cdn = "https://kf3-prod-cdn.kf3.wayi.com.tw/" server_app = "https://kf3-prod-app.kf3.wayi.com.tw/" async with aiohttp.ClientSession() as session: param = encode(json.dumps({"version":"1.0.0","dmm_viewer_id":0,"platform":1})) request = await download_bytes(server_app + "paradesv/common/GetUrl.do?param=" + param, session) result = gzip.decompress(request) response = json.loads(result) asset_bundle_url = response["asset_bundle_url"] urlName = asset_bundle_url.split("-")[2] print("downloading from", server_app) downloaded_cache = await download_cache("TW", server_app, server_cdn) downloaded_files = await download_files("TW", asset_bundle_url, "Android") # if downloaded_cache != [] and downloaded_cache != None: # with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_cache.json", "wt", encoding="utf-8") as file: # json.dump(downloaded_cache, file, ensure_ascii=False, indent=1) # if downloaded_files != [] and downloaded_files != None: # with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_files.json", "wt", encoding="utf-8") as file: # json.dump(downloaded_files, file, ensure_ascii=False, indent=1) if __name__ == "__main__": asyncio.run(manual()) #asyncio.run(convert_files())