import base64 import os import gzip import json import asyncio import hashlib import platform import aiohttp from UnityPy import enums from ..Shared.utility import divide_chunks from ..Shared.downloading import download_bytes, download_text from ..Shared.convert import convert, extract servers = [ "https://parade-mobile-stg-app.kemono-friends-3.jp/", "https://parade-mobile-develop01-app.kemono-friends-3.jp/", #"https://parade-mobile-develop02-app.kemono-friends-3.jp/paradesv/", #"https://parade-mobile-develop03-app.kemono-friends-3.jp/paradesv/", #"https://parade-mobile-develop04-app.kemono-friends-3.jp/paradesv/", ] def decode(data): # cut off the md5 checksum at the end and the four bytes at the start hash = bytearray.fromhex(data[:-(2*16)][2*4:]) key = hashlib.md5(bytearray.fromhex(data[:2*4])).digest() # md5 the key # xor with the key for i in range(0, len(hash)): hash[i] = hash[i] ^ key[i % (len(key))] return hash.decode("utf-8") def encode(text): hashed = hashlib.md5(text.encode()).digest() checksum = hashlib.md5((text + "DARAPAB ").encode()).digest() key = hashlib.md5(hashed[:4]).digest() # md5 the key # xor the data with the key text = bytearray(text.encode()) for i in range(0, len(text)): text[i] = text[i] ^ key[i % (len(key))] return hashed[:4].hex() + text.hex() + checksum.hex() async def download_cache(server_name, server : str): session = aiohttp.ClientSession() downloaded_files = [] if platform.system() == "Windows": path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\cache\\" else: path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/cache/" os.makedirs(path, exist_ok=True) new_mst_ver = {} old_mst_ver = {} file_path_mst = path + "mstVersion.txt" if os.path.exists(file_path_mst): with open(file_path_mst, "rt", encoding="utf-8") as file: old_mst_ver = dict([(entry["type"], entry["version"]) for entry in json.load(file)]) param = encode(json.dumps({'dmm_viewer_id':0})) request = await download_bytes(server + "paradesv/common/MstVersion.do?param=" + param, session) result = gzip.decompress(request) new_mst = json.loads(result) new_mst_ver = dict([(entry["type"], entry["version"]) for entry in new_mst["mst_ver"]]) for key in new_mst_ver: file_path_gzip = path + key + ".d" file_path_json = path + key + ".json" if os.path.exists(file_path_gzip) and os.path.exists(file_path_json) and key in old_mst_ver: if new_mst_ver[key] == old_mst_ver[key]: continue downloaded_files.append(key) param = encode(json.dumps({'type':key, 'dmm_viewer_id':0})) request = await download_bytes(server + "paradesv/common/MstData.do?param=" + param, session) result = gzip.decompress(request) response = json.loads(result) data = base64.b64decode(response["data"]) with open(file_path_gzip, "wb") as file: file.write(data) with open(file_path_json, "wt", encoding="utf-8") as out_file: data = gzip.decompress(data) data = json.loads(data) # if key == "GACHA_DATA": # download_banners(data, server_name) json.dump(data, out_file, ensure_ascii=False, indent=1) old_mst_ver[key] = new_mst_ver[key] with open(file_path_mst, "wt", encoding="utf-8") as file: new_json = [{"type":type, "version":version} for type, version in zip(old_mst_ver.keys(), old_mst_ver.values())] json.dump(new_json, file, ensure_ascii=False) await session.close() return downloaded_files async def download_banners(gacha_data, server_name, session): path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/banners/" os.makedirs(path, exist_ok=True) for entry in gacha_data: banner_name = entry["banner"] if banner_name == "" or banner_name == None: continue banner_name += ".png" file_path = path + banner_name if os.path.exists(file_path): continue file_url = "https://parade-mobile-prod-cdn.kemono-friends-3.jp/Texture2D/GachaTop/" + banner_name file_url_alt = "https://parade-mobile-develop01-app.kemono-friends-3.jp/Texture2D/GachaTop/" + banner_name status = 0 async with session.get(file_url) as resp: response = await resp.read() status = resp.status if status != 200: async with session.get(file_url_alt) as resp: response = await resp.read() status = resp.status if status != 200: continue with open(file_path, "wb") as file: file.write(response) async def download_files(server_name, asset_bundle_url, srv_platform : str): def parse_ab_list(filecontent : str): out = {} lines = filecontent.replace("\r\n", "\n").split('\n') for line in lines: split = line.split('\t') if len(split) > 1: out[split[0]] = split[-2] return out async def download_file(url_assets : str, file_name : str, download_path : str, session : aiohttp.ClientSession): data = await download_bytes(url_assets + file_name, session) if data != None: with open(download_path + file_name, "wb") as file: file.write(data) if server_name == "develop01": convert_path = f"/var/www/html/Katworks/KF/assets/KF3/WebGL/assets/" + file_name extract_path = f"/var/www/html/Katworks/KF/assets/KF3/extracted/" try: convert(data, convert_path, enums.BuildTarget.WebGL) except: with open(convert_path, "wb") as file: file.write(data) if file_name.endswith(".png"): extract(data, extract_path) session = aiohttp.ClientSession() path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/assets/{srv_platform}/" os.makedirs(path, exist_ok=True) files_to_download = [] url_base = asset_bundle_url + "/" + srv_platform + "/1.0.0/ja/" url_list = url_base + "ab_list.txt" url_env = url_base + "ab_env.txt" url_assets = url_base + "assets/" file_path_env = path + "ab_env.txt" file_path_list = path + "ab_list.txt" old_ab_env = "" if os.path.exists(file_path_env): with open(file_path_env, "rt") as file: old_ab_env = file.read() new_ab_env = await download_text(url_env, session) if new_ab_env != old_ab_env: old_ab_list = {} if os.path.exists(file_path_list): with open(file_path_list, "rt") as file: old_ab_list = parse_ab_list(file.read()) new_ab_list_file = await download_text(url_list, session) new_ab_list = parse_ab_list(new_ab_list_file) for key in new_ab_list: file_path = path + key if os.path.exists(file_path) and key in old_ab_list: if new_ab_list[key] == old_ab_list[key]: continue files_to_download.append(key) chunked_files_to_download = list(divide_chunks(files_to_download, 5)) for chunk in chunked_files_to_download: tasks = [asyncio.create_task(download_file(url_assets, file, path, session)) for file in chunk] await asyncio.wait(tasks) print(chunk) print() with open(file_path_env, "wt", encoding="utf-8") as file: file.write(new_ab_env) with open(file_path_list, "wt", encoding="utf-8") as file: file.write(new_ab_list_file) await session.close() return files_to_download async def convert_files(): directory = f"/var/www/html/Katworks/KF/assets/KF3/develop01/assets/Windows/" with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_dev_files.json", "rt", encoding="utf-8") as file: files_to_convert = json.load(file) for file_name in os.listdir(directory): if file_name not in files_to_convert: continue f = os.path.join(directory, file_name) if not os.path.isfile(f): return convert_path = f"/var/www/html/Katworks/KF/assets/KF3/WebGL/assets/" + file_name try: print(f) convert(f, convert_path, enums.BuildTarget.WebGL) except: print("Conversion failed", f) async def manual(): # session = aiohttp.ClientSession() # path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/cache/" # await session.close() # return downloaded_cache = {} downloaded_files = {} async with aiohttp.ClientSession() as session: param = encode(json.dumps({"version":"1.0.0","dmm_viewer_id":0,"platform":1})) request = await download_bytes(servers[0] + "paradesv/common/GetUrl.do?param=" + param, session) result = gzip.decompress(request) response = json.loads(result) asset_bundle_url = response["asset_bundle_url"] urlName = asset_bundle_url.split("-")[2] print("downloading from", servers[0]) downloaded_cache = await download_cache(urlName, servers[0]) downloaded_files = await download_files(urlName, asset_bundle_url, "Windows") if downloaded_cache != [] and downloaded_cache != None: with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_cache.json", "wt", encoding="utf-8") as file: json.dump(downloaded_cache, file, ensure_ascii=False, indent=1) if downloaded_files != [] and downloaded_files != None: with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_files.json", "wt", encoding="utf-8") as file: json.dump(downloaded_files, file, ensure_ascii=False, indent=1) print("downloading from", servers[1]) asset_bundle_url = "https://parade-mobile-develop01-app.kemono-friends-3.jp/AssetBundles/0.0.0/latest" urlName = asset_bundle_url.split("-")[2] downloaded_cache = await download_cache(urlName, servers[1]) downloaded_files = await download_files(urlName, asset_bundle_url, "Windows") if downloaded_cache != [] and downloaded_cache != None: with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_dev_cache.json", "wt", encoding="utf-8") as file: json.dump(downloaded_cache, file, ensure_ascii=False, indent=1) if downloaded_files != [] and downloaded_files != None: with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_dev_files.json", "wt", encoding="utf-8") as file: json.dump(downloaded_files, file, ensure_ascii=False, indent=1) if __name__ == "__main__": asyncio.run(manual()) #asyncio.run(convert_files())