375 lines
14 KiB
Python
375 lines
14 KiB
Python
|
|
import base64
|
|
import os
|
|
import gzip
|
|
import json
|
|
import asyncio
|
|
import hashlib
|
|
import platform
|
|
|
|
import aiohttp
|
|
from UnityPy import enums
|
|
import requests
|
|
|
|
from ..Shared.utility import divide_chunks
|
|
from ..Shared.downloading import download_bytes, download_text
|
|
from ..Shared.convert import convert, extract
|
|
|
|
servers = [
|
|
"https://parade-mobile-stg-app.kemono-friends-3.jp/",
|
|
"https://parade-mobile-develop01-app.kemono-friends-3.jp/",
|
|
#"https://parade-mobile-develop02-app.kemono-friends-3.jp/paradesv/",
|
|
#"https://parade-mobile-develop03-app.kemono-friends-3.jp/paradesv/",
|
|
#"https://parade-mobile-develop04-app.kemono-friends-3.jp/paradesv/",
|
|
]
|
|
|
|
def decode(data):
|
|
# cut off the md5 checksum at the end and the four bytes at the start
|
|
hash = bytearray.fromhex(data[:-(2*16)][2*4:])
|
|
key = hashlib.md5(bytearray.fromhex(data[:2*4])).digest() # md5 the key
|
|
# xor with the key
|
|
for i in range(0, len(hash)):
|
|
hash[i] = hash[i] ^ key[i % (len(key))]
|
|
return hash.decode("utf-8")
|
|
|
|
def encode(text):
|
|
hashed = hashlib.md5(text.encode()).digest()
|
|
checksum = hashlib.md5((text + "DARAPAB ").encode()).digest()
|
|
key = hashlib.md5(hashed[:4]).digest() # md5 the key
|
|
# xor the data with the key
|
|
text = bytearray(text.encode())
|
|
for i in range(0, len(text)):
|
|
text[i] = text[i] ^ key[i % (len(key))]
|
|
return hashed[:4].hex() + text.hex() + checksum.hex()
|
|
|
|
async def download_cache(server_name, server_app : str, server_cdn : str = None):
|
|
session = aiohttp.ClientSession()
|
|
downloaded_files = []
|
|
|
|
if platform.system() == "Windows":
|
|
path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\cache\\"
|
|
else:
|
|
path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/cache/"
|
|
|
|
os.makedirs(path, exist_ok=True)
|
|
|
|
new_mst_ver = {}
|
|
old_mst_ver = {}
|
|
file_path_mst = path + "mstVersion.txt"
|
|
if os.path.exists(file_path_mst):
|
|
with open(file_path_mst, "rt", encoding="utf-8") as file:
|
|
old_mst_ver = dict([(entry["type"], entry["version"]) for entry in json.load(file)])
|
|
|
|
param = encode(json.dumps({'dmm_viewer_id':0}))
|
|
request = await download_bytes(server_app + "paradesv/common/MstVersion.do?param=" + param, session)
|
|
result = gzip.decompress(request)
|
|
new_mst = json.loads(result)
|
|
new_mst_ver = dict([(entry["type"], entry["version"]) for entry in new_mst["mst_ver"]])
|
|
|
|
for key in new_mst_ver:
|
|
file_path_gzip = path + key + ".d"
|
|
file_path_json = path + key + ".json"
|
|
if os.path.exists(file_path_gzip) and os.path.exists(file_path_json) and key in old_mst_ver:
|
|
if new_mst_ver[key] == old_mst_ver[key]:
|
|
continue
|
|
|
|
downloaded_files.append(key)
|
|
param = encode(json.dumps({'type':key, 'dmm_viewer_id':0}))
|
|
request = await download_bytes(server_app + "paradesv/common/MstData.do?param=" + param, session)
|
|
result = gzip.decompress(request)
|
|
response = json.loads(result)
|
|
data = base64.b64decode(response["data"])
|
|
with open(file_path_gzip, "wb") as file:
|
|
file.write(data)
|
|
with open(file_path_json, "wt", encoding="utf-8") as out_file:
|
|
data = gzip.decompress(data)
|
|
data = json.loads(data)
|
|
if key == "GACHA_DATA":
|
|
await download_banners(data, server_name, server_app, server_cdn, session)
|
|
if key == "BANNER_DATA":
|
|
await download_banner_data(data, server_name, server_cdn, session)
|
|
if key == "EVENT_BANNER_DATA":
|
|
await download_event_banner_data(data, server_name, server_cdn, session)
|
|
if key == "EVENT_DATA":
|
|
await download_event_data(data, server_name, server_cdn, session)
|
|
json.dump(data, out_file, ensure_ascii=False, indent=1)
|
|
|
|
old_mst_ver[key] = new_mst_ver[key]
|
|
|
|
with open(file_path_mst, "wt", encoding="utf-8") as file:
|
|
new_json = [{"type":type, "version":version} for type, version in zip(old_mst_ver.keys(), old_mst_ver.values())]
|
|
json.dump(new_json, file, ensure_ascii=False)
|
|
|
|
await session.close()
|
|
return downloaded_files
|
|
|
|
async def download_banners(gacha_data, server_name, server_app, server_cdn, session):
|
|
path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\banners\\"
|
|
os.makedirs(path, exist_ok=True)
|
|
|
|
async def download_banner(entry):
|
|
banner_name = entry["banner"]
|
|
if banner_name == "" or banner_name == None:
|
|
return
|
|
banner_name += ".png"
|
|
file_path = path + banner_name
|
|
if os.path.exists(file_path):
|
|
return
|
|
|
|
file_url_alt = f"{server_cdn}Texture2D/GachaTop/{banner_name}"
|
|
file_url = f"{server_app}Texture2D/GachaTop/{banner_name}"
|
|
|
|
status = 0
|
|
async with session.get(file_url) as resp:
|
|
response = await resp.read()
|
|
status = resp.status
|
|
if status != 200:
|
|
async with session.get(file_url_alt) as resp:
|
|
response = await resp.read()
|
|
status = resp.status
|
|
if status != 200: return
|
|
|
|
with open(file_path, "wb") as file:
|
|
file.write(response)
|
|
|
|
chunked_files_to_download = list(divide_chunks(gacha_data, 20))
|
|
|
|
for chunk in chunked_files_to_download:
|
|
tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk]
|
|
await asyncio.wait(tasks)
|
|
|
|
async def download_banner_data(gacha_data, server_name, server_cdn, session : aiohttp.ClientSession):
|
|
path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\"
|
|
|
|
async def download_banner(entry):
|
|
name = entry["bannerName"]
|
|
if name == "" or name == None:
|
|
return
|
|
bannerImagePath = "Texture2D/HomeBanner/home_banner_" + name + ".png"
|
|
bannerImagePathByQuestTop = "Texture2D/QuestTop/questtop_banner_" + name + ".png"
|
|
bannerImagePathEvent = "Texture2D/EventTop/eventtop_banner_" + name + ".png"
|
|
banners = [bannerImagePath, bannerImagePathByQuestTop, bannerImagePathEvent]
|
|
|
|
for banner in banners:
|
|
file_path = path + banner
|
|
if os.path.exists(file_path):
|
|
continue
|
|
|
|
fileDir = "/".join(file_path.split("/")[:-1])
|
|
os.makedirs(fileDir, exist_ok=True)
|
|
file_url = f"{server_cdn}{banner}"
|
|
|
|
try:
|
|
with open(file_path, 'wb') as out_file:
|
|
req = requests.get(file_url, stream=True)
|
|
if req.status_code != 200: raise Exception("lol")
|
|
out_file.write(req.content)
|
|
except Exception as ex:
|
|
os.remove(file_path)
|
|
print(ex)
|
|
continue
|
|
|
|
chunked_files_to_download = list(divide_chunks(gacha_data, 20))
|
|
|
|
for chunk in chunked_files_to_download:
|
|
tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk]
|
|
await asyncio.wait(tasks)
|
|
|
|
async def download_event_banner_data(gacha_data, server_name, server_cdn, session : aiohttp.ClientSession):
|
|
path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\"
|
|
|
|
async def download_banner(entry):
|
|
name = entry["bannerName"]
|
|
if name == "" or name == None:
|
|
return
|
|
banner = "Texture2D/HomeBigBanner/home_bigbanner_" + name + ".png"
|
|
|
|
file_path = path + banner
|
|
if os.path.exists(file_path):
|
|
return
|
|
|
|
fileDir = "/".join(file_path.split("/")[:-1])
|
|
os.makedirs(fileDir, exist_ok=True)
|
|
file_url = f"{server_cdn}{banner}"
|
|
|
|
try:
|
|
with open(file_path, 'wb') as out_file:
|
|
req = requests.get(file_url, stream=True)
|
|
if req.status_code != 200: raise Exception("lol")
|
|
out_file.write(req.content)
|
|
except Exception as ex:
|
|
os.remove(file_path)
|
|
print(ex)
|
|
|
|
chunked_files_to_download = list(divide_chunks(gacha_data, 20))
|
|
|
|
for chunk in chunked_files_to_download:
|
|
tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk]
|
|
await asyncio.wait(tasks)
|
|
|
|
|
|
async def download_event_data(gacha_data, server_name, server_cdn, session : aiohttp.ClientSession):
|
|
path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\"
|
|
|
|
async def download_banner(entry):
|
|
name = entry["missionBannerFilename"]
|
|
if name == "" or name == None:
|
|
return
|
|
banner = "Texture2D/Mission/" + name + ".png"
|
|
|
|
file_path = path + banner
|
|
if os.path.exists(file_path):
|
|
return
|
|
|
|
fileDir = "/".join(file_path.split("/")[:-1])
|
|
os.makedirs(fileDir, exist_ok=True)
|
|
file_url = f"{server_cdn}{banner}"
|
|
|
|
try:
|
|
with open(file_path, 'wb') as out_file:
|
|
req = requests.get(file_url, stream=True)
|
|
if req.status_code != 200: raise Exception("lol")
|
|
out_file.write(req.content)
|
|
except Exception as ex:
|
|
os.remove(file_path)
|
|
print(ex)
|
|
|
|
chunked_files_to_download = list(divide_chunks(gacha_data, 20))
|
|
|
|
for chunk in chunked_files_to_download:
|
|
tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk]
|
|
await asyncio.wait(tasks)
|
|
|
|
async def download_files(server_name, asset_bundle_url, srv_platform : str):
|
|
def parse_ab_list(filecontent : str):
|
|
out = {}
|
|
lines = filecontent.replace("\r\n", "\n").split('\n')
|
|
for line in lines:
|
|
split = line.split('\t')
|
|
if len(split) > 1:
|
|
out[split[0]] = split[-2]
|
|
return out
|
|
|
|
async def download_file(url_assets : str, file_name : str, download_path : str, session : aiohttp.ClientSession):
|
|
data = await download_bytes(url_assets + file_name, session)
|
|
if data != None:
|
|
with open(download_path + file_name, "wb") as file:
|
|
file.write(data)
|
|
if server_name == "develop01":
|
|
convert_path = f"/var/www/html/Katworks/KF/assets/KF3/WebGL/assets/" + file_name
|
|
extract_path = f"/var/www/html/Katworks/KF/assets/KF3/extracted/"
|
|
try:
|
|
convert(data, convert_path, enums.BuildTarget.WebGL)
|
|
except:
|
|
with open(convert_path, "wb") as file:
|
|
file.write(data)
|
|
if file_name.endswith(".png"):
|
|
extract(data, extract_path)
|
|
|
|
session = aiohttp.ClientSession()
|
|
|
|
if platform.system() == "Windows":
|
|
path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\assets\\{srv_platform}\\"
|
|
else:
|
|
path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/assets/{srv_platform}/"
|
|
|
|
os.makedirs(path, exist_ok=True)
|
|
|
|
files_to_download = []
|
|
url_base = asset_bundle_url + "/" + srv_platform + "/1.0.0/ja/"
|
|
url_list = url_base + "ab_list.txt"
|
|
url_env = url_base + "ab_env.txt"
|
|
url_assets = url_base + "assets/"
|
|
|
|
file_path_env = path + "ab_env.txt"
|
|
file_path_list = path + "ab_list.txt"
|
|
old_ab_env = ""
|
|
|
|
if os.path.exists(file_path_env):
|
|
with open(file_path_env, "rt") as file:
|
|
old_ab_env = file.read()
|
|
|
|
new_ab_env = await download_text(url_env, session)
|
|
|
|
if new_ab_env != old_ab_env:
|
|
old_ab_list = {}
|
|
if os.path.exists(file_path_list):
|
|
with open(file_path_list, "rt") as file:
|
|
old_ab_list = parse_ab_list(file.read())
|
|
|
|
new_ab_list_file = await download_text(url_list, session)
|
|
new_ab_list = parse_ab_list(new_ab_list_file)
|
|
|
|
for key in new_ab_list:
|
|
file_path = path + key
|
|
if os.path.exists(file_path) and key in old_ab_list:
|
|
if new_ab_list[key] == old_ab_list[key]:
|
|
continue
|
|
|
|
files_to_download.append(key)
|
|
|
|
chunked_files_to_download = list(divide_chunks(files_to_download, 10))
|
|
|
|
for chunk in chunked_files_to_download:
|
|
tasks = [asyncio.create_task(download_file(url_assets, file, path, session)) for file in chunk]
|
|
await asyncio.wait(tasks)
|
|
print(chunk)
|
|
print()
|
|
|
|
with open(file_path_env, "wt", encoding="utf-8") as file:
|
|
file.write(new_ab_env)
|
|
with open(file_path_list, "wt", encoding="utf-8") as file:
|
|
file.write(new_ab_list_file)
|
|
|
|
await session.close()
|
|
return files_to_download
|
|
|
|
async def convert_files():
|
|
directory = f"/var/www/html/Katworks/KF/assets/KF3/develop01/assets/Windows/"
|
|
with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_dev_files.json", "rt", encoding="utf-8") as file:
|
|
files_to_convert = json.load(file)
|
|
for file_name in os.listdir(directory):
|
|
if file_name not in files_to_convert:
|
|
continue
|
|
|
|
f = os.path.join(directory, file_name)
|
|
|
|
if not os.path.isfile(f):
|
|
return
|
|
|
|
convert_path = f"/var/www/html/Katworks/KF/assets/KF3/WebGL/assets/" + file_name
|
|
try:
|
|
print(f)
|
|
convert(f, convert_path, enums.BuildTarget.WebGL)
|
|
except:
|
|
print("Conversion failed", f)
|
|
|
|
async def manual():
|
|
downloaded_cache = {}
|
|
downloaded_files = {}
|
|
|
|
server_cdn = "https://kf3-prod-cdn.kf3.wayi.com.tw/"
|
|
server_app = "https://kf3-prod-app.kf3.wayi.com.tw/"
|
|
|
|
async with aiohttp.ClientSession() as session:
|
|
param = encode(json.dumps({"version":"1.0.0","dmm_viewer_id":0,"platform":1}))
|
|
request = await download_bytes(server_app + "paradesv/common/GetUrl.do?param=" + param, session)
|
|
result = gzip.decompress(request)
|
|
response = json.loads(result)
|
|
asset_bundle_url = response["asset_bundle_url"]
|
|
urlName = asset_bundle_url.split("-")[2]
|
|
|
|
print("downloading from", server_app)
|
|
downloaded_cache = await download_cache("TW", server_app, server_cdn)
|
|
downloaded_files = await download_files("TW", asset_bundle_url, "Android")
|
|
|
|
# if downloaded_cache != [] and downloaded_cache != None:
|
|
# with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_cache.json", "wt", encoding="utf-8") as file:
|
|
# json.dump(downloaded_cache, file, ensure_ascii=False, indent=1)
|
|
# if downloaded_files != [] and downloaded_files != None:
|
|
# with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_files.json", "wt", encoding="utf-8") as file:
|
|
# json.dump(downloaded_files, file, ensure_ascii=False, indent=1)
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(manual())
|
|
#asyncio.run(convert_files()) |