KemoFureApi/modules/KF3/downloader.py

import base64
import os
import gzip
import json
import asyncio
import hashlib
import platform

import aiohttp
from UnityPy import enums

from ..Shared.utility import divide_chunks
from ..Shared.downloading import download_bytes, download_text
from ..Shared.convert import convert, extract

servers = [
    "https://parade-mobile-stg-app.kemono-friends-3.jp/",
    "https://parade-mobile-develop01-app.kemono-friends-3.jp/",
    #"https://parade-mobile-develop02-app.kemono-friends-3.jp/paradesv/",
    #"https://parade-mobile-develop03-app.kemono-friends-3.jp/paradesv/",
    #"https://parade-mobile-develop04-app.kemono-friends-3.jp/paradesv/",
]

def decode(data):
     # cut off the md5 checksum at the end and the four bytes at the start
    hash = bytearray.fromhex(data[:-(2*16)][2*4:]) 
    key = hashlib.md5(bytearray.fromhex(data[:2*4])).digest() # md5 the key
    # xor with the key
    for i in range(0, len(hash)):
        hash[i] = hash[i] ^ key[i % (len(key))] 
    return hash.decode("utf-8")

def encode(text):
    hashed = hashlib.md5(text.encode()).digest()
    checksum = hashlib.md5((text + "DARAPAB ").encode()).digest()
    key = hashlib.md5(hashed[:4]).digest() # md5 the key
    # xor the data with the key
    text = bytearray(text.encode())
    for i in range(0, len(text)):
        text[i] = text[i] ^ key[i % (len(key))] 
    return hashed[:4].hex() + text.hex() + checksum.hex()

async def download_cache(server_name, server : str):
    session = aiohttp.ClientSession()
    downloaded_files = []
    
    if platform.system() == "Windows":
        path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\cache\\"
    else:
        path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/cache/"
   
    os.makedirs(path, exist_ok=True)

    new_mst_ver = {}
    old_mst_ver = {}
    file_path_mst = path + "mstVersion.txt"
    if os.path.exists(file_path_mst):
        with open(file_path_mst, "rt", encoding="utf-8") as file:
            old_mst_ver = dict([(entry["type"], entry["version"]) for entry in json.load(file)])
    
    param   = encode(json.dumps({'dmm_viewer_id':0}))
    request = await download_bytes(server + "paradesv/common/MstVersion.do?param=" + param, session)
    result  = gzip.decompress(request)
    new_mst = json.loads(result)
    new_mst_ver = dict([(entry["type"], entry["version"]) for entry in new_mst["mst_ver"]])

    for key in new_mst_ver:
        file_path_gzip = path + key + ".d"
        file_path_json = path + key + ".json"
        if os.path.exists(file_path_gzip) and os.path.exists(file_path_json) and key in old_mst_ver:
            if new_mst_ver[key] == old_mst_ver[key]:
                continue

        downloaded_files.append(key)
        param = encode(json.dumps({'type':key, 'dmm_viewer_id':0}))        
        request = await download_bytes(server + "paradesv/common/MstData.do?param=" + param, session)
        result  = gzip.decompress(request)
        response = json.loads(result)
        data = base64.b64decode(response["data"])
        with open(file_path_gzip, "wb") as file:
            file.write(data)
        with open(file_path_json, "wt", encoding="utf-8") as out_file:
            data = gzip.decompress(data)
            json.dump(json.loads(data), out_file, ensure_ascii=False, indent=1)

        old_mst_ver[key] = new_mst_ver[key]

    with open(file_path_mst, "wt", encoding="utf-8") as file:
        new_json = [{"type":type, "version":version} for type, version in zip(old_mst_ver.keys(), old_mst_ver.values())]
        json.dump(new_json, file, ensure_ascii=False)

    await session.close()
    return downloaded_files

async def download_files(server_name, asset_bundle_url, srv_platform : str):
    def parse_ab_list(filecontent : str):
        out = {}
        lines = filecontent.replace("\r\n", "\n").split('\n')
        for line in lines:
            split = line.split('\t')
            if len(split) > 1:
                out[split[0]] = split[-2]
        return out
    
    async def download_file(url_assets : str, file_name : str, download_path : str, session : aiohttp.ClientSession):
        data = await download_bytes(url_assets + file_name, session)
        if data != None:
            with open(download_path + file_name, "wb") as file:
                file.write(data)
            if server_name == "develop01":
                convert_path = f"/var/www/html/Katworks/KF/assets/KF3/WebGL/assets/" + file_name
                extract_path = f"/var/www/html/Katworks/KF/assets/KF3/extracted/"
                try:
                    convert(data, convert_path, enums.BuildTarget.WebGL)
                except:
                    with open(convert_path, "wb") as file:
                        file.write(data)
                if file_name.endswith(".png"):
                    extract(data, extract_path)

    session = aiohttp.ClientSession()
    path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/assets/{srv_platform}/"

    os.makedirs(path, exist_ok=True)

    files_to_download = []
    url_base = asset_bundle_url + "/" + srv_platform + "/1.0.0/ja/"
    url_list = url_base + "ab_list.txt"
    url_env = url_base + "ab_env.txt"
    url_assets = url_base + "assets/"

    file_path_env = path + "ab_env.txt"
    file_path_list = path + "ab_list.txt"
    old_ab_env = ""

    if os.path.exists(file_path_env):
        with open(file_path_env, "rt") as file:
            old_ab_env = file.read()

    new_ab_env = await download_text(url_env, session)

    if new_ab_env != old_ab_env:
        old_ab_list = {}
        if os.path.exists(file_path_list):
            with open(file_path_list, "rt") as file:
                old_ab_list = parse_ab_list(file.read())
        
        new_ab_list_file = await download_text(url_list, session)
        new_ab_list = parse_ab_list(new_ab_list_file)
        
        for key in new_ab_list:
            file_path = path + key
            if os.path.exists(file_path) and key in old_ab_list:
                if new_ab_list[key] == old_ab_list[key]:
                    continue
            
            files_to_download.append(key)

        chunked_files_to_download = list(divide_chunks(files_to_download, 5)) 
        
        for chunk in chunked_files_to_download:
            tasks = [asyncio.create_task(download_file(url_assets, file, path, session)) for file in chunk]
            await asyncio.wait(tasks)
            print(chunk)
            print()

        with open(file_path_env, "wt", encoding="utf-8") as file:
            file.write(new_ab_env)
        with open(file_path_list, "wt", encoding="utf-8") as file:
            file.write(new_ab_list_file)
    
    await session.close()
    return files_to_download

async def convert_files():
    directory = f"/var/www/html/Katworks/KF/assets/KF3/develop01/assets/Windows/"
    with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_dev_files.json", "rt", encoding="utf-8") as file:
        files_to_convert = json.load(file)
    for file_name in os.listdir(directory):
        if file_name not in files_to_convert:
            continue
        
        f = os.path.join(directory, file_name)

        if not os.path.isfile(f):
            return
        
        convert_path = f"/var/www/html/Katworks/KF/assets/KF3/WebGL/assets/" + file_name
        try:
            print(f)
            convert(f, convert_path, enums.BuildTarget.WebGL)
        except:
            print("Conversion failed", f)

async def manual():
    downloaded_cache = {}
    downloaded_files = {}
    
    async with aiohttp.ClientSession() as session:
        param = encode(json.dumps({"version":"1.0.0","dmm_viewer_id":0,"platform":1}))
        request = await download_bytes(servers[0] + "paradesv/common/GetUrl.do?param=" + param, session)
        result  = gzip.decompress(request)
        response = json.loads(result)
        asset_bundle_url = response["asset_bundle_url"]
        urlName = asset_bundle_url.split("-")[2]

    print("downloading from", servers[0])
    downloaded_cache = await download_cache(urlName, servers[0])
    downloaded_files = await download_files(urlName, asset_bundle_url, "Windows")

    if downloaded_cache != [] and downloaded_cache != None:
        with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_cache.json", "wt", encoding="utf-8") as file:
            json.dump(downloaded_cache, file, ensure_ascii=False, indent=1)
    if downloaded_files != [] and downloaded_files != None:
        with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_files.json", "wt", encoding="utf-8") as file:
            json.dump(downloaded_files, file, ensure_ascii=False, indent=1)

    print("downloading from", servers[1])
    asset_bundle_url = "https://parade-mobile-develop01-app.kemono-friends-3.jp/AssetBundles/0.0.0/latest"
    urlName = asset_bundle_url.split("-")[2]
    downloaded_cache = await download_cache(urlName, servers[1])
    downloaded_files = await download_files(urlName, asset_bundle_url, "Windows")

    if downloaded_cache != [] and downloaded_cache != None:
        with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_dev_cache.json", "wt", encoding="utf-8") as file:
            json.dump(downloaded_cache, file, ensure_ascii=False, indent=1)
    if downloaded_files != [] and downloaded_files != None:
        with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_dev_files.json", "wt", encoding="utf-8") as file:
            json.dump(downloaded_files, file, ensure_ascii=False, indent=1)

if __name__ == "__main__":
    asyncio.run(manual())
    #asyncio.run(convert_files())