katboi01
/
KemoFureApi
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

undo downloader changes

This commit is contained in:
katboi01 2025-03-16 20:41:53 +01:00
parent b5d63d1d00
commit b3aa760f3d
1 changed files with 52 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

201
modules/KF3/downloader.py
View File

@ -9,7 +9,6 @@ import platform
import aiohttp import aiohttp
from UnityPy import enums from UnityPy import enums
import requests
from ..Shared.utility import divide_chunks from ..Shared.utility import divide_chunks
from ..Shared.downloading import download_bytes, download_text from ..Shared.downloading import download_bytes, download_text
@ -42,7 +41,7 @@ def encode(text):
text[i] = text[i] ^ key[i % (len(key))] text[i] = text[i] ^ key[i % (len(key))]
return hashed[:4].hex() + text.hex() + checksum.hex() return hashed[:4].hex() + text.hex() + checksum.hex()
async def download_cache(server_name, server_app : str, server_cdn : str = None): async def download_cache(server_name, server : str):
session = aiohttp.ClientSession() session = aiohttp.ClientSession()
downloaded_files = [] downloaded_files = []
@ -61,7 +60,7 @@ async def download_cache(server_name, server_app : str, server_cdn : str = None)
old_mst_ver = dict([(entry["type"], entry["version"]) for entry in json.load(file)]) old_mst_ver = dict([(entry["type"], entry["version"]) for entry in json.load(file)])
param = encode(json.dumps({'dmm_viewer_id':0})) param = encode(json.dumps({'dmm_viewer_id':0}))
request = await download_bytes(server_app + "paradesv/common/MstVersion.do?param=" + param, session) request = await download_bytes(server + "paradesv/common/MstVersion.do?param=" + param, session)
result = gzip.decompress(request) result = gzip.decompress(request)
new_mst = json.loads(result) new_mst = json.loads(result)
new_mst_ver = dict([(entry["type"], entry["version"]) for entry in new_mst["mst_ver"]]) new_mst_ver = dict([(entry["type"], entry["version"]) for entry in new_mst["mst_ver"]])
@ -75,7 +74,7 @@ async def download_cache(server_name, server_app : str, server_cdn : str = None)
downloaded_files.append(key) downloaded_files.append(key)
param = encode(json.dumps({'type':key, 'dmm_viewer_id':0})) param = encode(json.dumps({'type':key, 'dmm_viewer_id':0}))
request = await download_bytes(server_app + "paradesv/common/MstData.do?param=" + param, session) request = await download_bytes(server + "paradesv/common/MstData.do?param=" + param, session)
result = gzip.decompress(request) result = gzip.decompress(request)
response = json.loads(result) response = json.loads(result)
data = base64.b64decode(response["data"]) data = base64.b64decode(response["data"])
@ -84,14 +83,8 @@ async def download_cache(server_name, server_app : str, server_cdn : str = None)
with open(file_path_json, "wt", encoding="utf-8") as out_file: with open(file_path_json, "wt", encoding="utf-8") as out_file:
data = gzip.decompress(data) data = gzip.decompress(data)
data = json.loads(data) data = json.loads(data)
if key == "GACHA_DATA": # if key == "GACHA_DATA":
await download_banners(data, server_name, server_app, server_cdn, session) # download_banners(data, server_name)
if key == "BANNER_DATA":
await download_banner_data(data, server_name, server_cdn, session)
if key == "EVENT_BANNER_DATA":
await download_event_banner_data(data, server_name, server_cdn, session)
if key == "EVENT_DATA":
await download_event_data(data, server_name, server_cdn, session)
json.dump(data, out_file, ensure_ascii=False, indent=1) json.dump(data, out_file, ensure_ascii=False, indent=1)
old_mst_ver[key] = new_mst_ver[key] old_mst_ver[key] = new_mst_ver[key]
@ -103,21 +96,21 @@ async def download_cache(server_name, server_app : str, server_cdn : str = None)
await session.close() await session.close()
return downloaded_files return downloaded_files
async def download_banners(gacha_data, server_name, server_app, server_cdn, session): async def download_banners(gacha_data, server_name, session):
path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\banners\\" path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/banners/"
os.makedirs(path, exist_ok=True) os.makedirs(path, exist_ok=True)
async def download_banner(entry): for entry in gacha_data:
banner_name = entry["banner"] banner_name = entry["banner"]
if banner_name == "" or banner_name == None: if banner_name == "" or banner_name == None:
return continue
banner_name += ".png" banner_name += ".png"
file_path = path + banner_name file_path = path + banner_name
if os.path.exists(file_path): if os.path.exists(file_path):
return continue
file_url_alt = f"{server_cdn}Texture2D/GachaTop/{banner_name}" file_url = "https://parade-mobile-prod-cdn.kemono-friends-3.jp/Texture2D/GachaTop/" + banner_name
file_url = f"{server_app}Texture2D/GachaTop/{banner_name}" file_url_alt = "https://parade-mobile-develop01-app.kemono-friends-3.jp/Texture2D/GachaTop/" + banner_name
status = 0 status = 0
async with session.get(file_url) as resp: async with session.get(file_url) as resp:
@ -127,118 +120,11 @@ async def download_banners(gacha_data, server_name, server_app, server_cdn, sess
async with session.get(file_url_alt) as resp: async with session.get(file_url_alt) as resp:
response = await resp.read() response = await resp.read()
status = resp.status status = resp.status
if status != 200: return if status != 200: continue
with open(file_path, "wb") as file: with open(file_path, "wb") as file:
file.write(response) file.write(response)
chunked_files_to_download = list(divide_chunks(gacha_data, 20))
for chunk in chunked_files_to_download:
tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk]
await asyncio.wait(tasks)
async def download_banner_data(gacha_data, server_name, server_cdn, session : aiohttp.ClientSession):
path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\"
async def download_banner(entry):
name = entry["bannerName"]
if name == "" or name == None:
return
bannerImagePath = "Texture2D/HomeBanner/home_banner_" + name + ".png"
bannerImagePathByQuestTop = "Texture2D/QuestTop/questtop_banner_" + name + ".png"
bannerImagePathEvent = "Texture2D/EventTop/eventtop_banner_" + name + ".png"
banners = [bannerImagePath, bannerImagePathByQuestTop, bannerImagePathEvent]
for banner in banners:
file_path = path + banner
if os.path.exists(file_path):
continue
fileDir = "/".join(file_path.split("/")[:-1])
os.makedirs(fileDir, exist_ok=True)
file_url = f"{server_cdn}{banner}"
try:
with open(file_path, 'wb') as out_file:
req = requests.get(file_url, stream=True)
if req.status_code != 200: raise Exception("lol")
out_file.write(req.content)
except Exception as ex:
os.remove(file_path)
print(ex)
continue
chunked_files_to_download = list(divide_chunks(gacha_data, 20))
for chunk in chunked_files_to_download:
tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk]
await asyncio.wait(tasks)
async def download_event_banner_data(gacha_data, server_name, server_cdn, session : aiohttp.ClientSession):
path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\"
async def download_banner(entry):
name = entry["bannerName"]
if name == "" or name == None:
return
banner = "Texture2D/HomeBigBanner/home_bigbanner_" + name + ".png"
file_path = path + banner
if os.path.exists(file_path):
return
fileDir = "/".join(file_path.split("/")[:-1])
os.makedirs(fileDir, exist_ok=True)
file_url = f"{server_cdn}{banner}"
try:
with open(file_path, 'wb') as out_file:
req = requests.get(file_url, stream=True)
if req.status_code != 200: raise Exception("lol")
out_file.write(req.content)
except Exception as ex:
os.remove(file_path)
print(ex)
chunked_files_to_download = list(divide_chunks(gacha_data, 20))
for chunk in chunked_files_to_download:
tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk]
await asyncio.wait(tasks)
async def download_event_data(gacha_data, server_name, server_cdn, session : aiohttp.ClientSession):
path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\"
async def download_banner(entry):
name = entry["missionBannerFilename"]
if name == "" or name == None:
return
banner = "Texture2D/Mission/" + name + ".png"
file_path = path + banner
if os.path.exists(file_path):
return
fileDir = "/".join(file_path.split("/")[:-1])
os.makedirs(fileDir, exist_ok=True)
file_url = f"{server_cdn}{banner}"
try:
with open(file_path, 'wb') as out_file:
req = requests.get(file_url, stream=True)
if req.status_code != 200: raise Exception("lol")
out_file.write(req.content)
except Exception as ex:
os.remove(file_path)
print(ex)
chunked_files_to_download = list(divide_chunks(gacha_data, 20))
for chunk in chunked_files_to_download:
tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk]
await asyncio.wait(tasks)
async def download_files(server_name, asset_bundle_url, srv_platform : str): async def download_files(server_name, asset_bundle_url, srv_platform : str):
def parse_ab_list(filecontent : str): def parse_ab_list(filecontent : str):
@ -267,10 +153,6 @@ async def download_files(server_name, asset_bundle_url, srv_platform : str):
extract(data, extract_path) extract(data, extract_path)
session = aiohttp.ClientSession() session = aiohttp.ClientSession()
if platform.system() == "Windows":
path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\assets\\{srv_platform}\\"
else:
path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/assets/{srv_platform}/" path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/assets/{srv_platform}/"
os.makedirs(path, exist_ok=True) os.makedirs(path, exist_ok=True)
@ -308,7 +190,7 @@ async def download_files(server_name, asset_bundle_url, srv_platform : str):
files_to_download.append(key) files_to_download.append(key)
chunked_files_to_download = list(divide_chunks(files_to_download, 10)) chunked_files_to_download = list(divide_chunks(files_to_download, 5))
for chunk in chunked_files_to_download: for chunk in chunked_files_to_download:
tasks = [asyncio.create_task(download_file(url_assets, file, path, session)) for file in chunk] tasks = [asyncio.create_task(download_file(url_assets, file, path, session)) for file in chunk]
@ -326,11 +208,11 @@ async def download_files(server_name, asset_bundle_url, srv_platform : str):
async def convert_files(): async def convert_files():
directory = f"/var/www/html/Katworks/KF/assets/KF3/develop01/assets/Windows/" directory = f"/var/www/html/Katworks/KF/assets/KF3/develop01/assets/Windows/"
with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_dev_files.json", "rt", encoding="utf-8") as file: # with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_dev_files.json", "rt", encoding="utf-8") as file:
files_to_convert = json.load(file) # files_to_convert = json.load(file)
for file_name in os.listdir(directory): for file_name in os.listdir(directory):
if file_name not in files_to_convert: # if file_name not in files_to_convert:
continue # continue
f = os.path.join(directory, file_name) f = os.path.join(directory, file_name)
@ -345,31 +227,50 @@ async def convert_files():
print("Conversion failed", f) print("Conversion failed", f)
async def manual(): async def manual():
# session = aiohttp.ClientSession()
# path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/cache/"
# await session.close()
# return
downloaded_cache = {} downloaded_cache = {}
downloaded_files = {} downloaded_files = {}
server_cdn = "https://kf3-prod-cdn.kf3.wayi.com.tw/"
server_app = "https://kf3-prod-app.kf3.wayi.com.tw/"
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
param = encode(json.dumps({"version":"1.0.0","dmm_viewer_id":0,"platform":1})) param = encode(json.dumps({"version":"1.0.0","dmm_viewer_id":0,"platform":1}))
request = await download_bytes(server_app + "paradesv/common/GetUrl.do?param=" + param, session) request = await download_bytes(servers[0] + "paradesv/common/GetUrl.do?param=" + param, session)
result = gzip.decompress(request) result = gzip.decompress(request)
response = json.loads(result) response = json.loads(result)
asset_bundle_url = response["asset_bundle_url"] asset_bundle_url = response["asset_bundle_url"]
urlName = asset_bundle_url.split("-")[2] urlName = asset_bundle_url.split("-")[2]
print("downloading from", server_app) print("downloading from", servers[0])
downloaded_cache = await download_cache("TW", server_app, server_cdn) downloaded_cache = await download_cache(urlName, servers[0])
downloaded_files = await download_files("TW", asset_bundle_url, "Android") downloaded_files = await download_files(urlName, asset_bundle_url, "Windows")
# if downloaded_cache != [] and downloaded_cache != None: if downloaded_cache != [] and downloaded_cache != None:
# with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_cache.json", "wt", encoding="utf-8") as file: with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_cache.json", "wt", encoding="utf-8") as file:
# json.dump(downloaded_cache, file, ensure_ascii=False, indent=1) json.dump(downloaded_cache, file, ensure_ascii=False, indent=1)
# if downloaded_files != [] and downloaded_files != None: if downloaded_files != [] and downloaded_files != None:
# with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_files.json", "wt", encoding="utf-8") as file: with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_files.json", "wt", encoding="utf-8") as file:
# json.dump(downloaded_files, file, ensure_ascii=False, indent=1) json.dump(downloaded_files, file, ensure_ascii=False, indent=1)
print("downloading from", servers[1])
asset_bundle_url = "https://parade-mobile-develop01-app.kemono-friends-3.jp/AssetBundles/0.0.0/latest"
urlName = asset_bundle_url.split("-")[2]
downloaded_cache = await download_cache(urlName, servers[1])
downloaded_files = await download_files(urlName, asset_bundle_url, "Windows")
if downloaded_cache != [] and downloaded_cache != None:
with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_dev_cache.json", "wt", encoding="utf-8") as file:
json.dump(downloaded_cache, file, ensure_ascii=False, indent=1)
if downloaded_files != [] and downloaded_files != None:
with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_dev_files.json", "wt", encoding="utf-8") as file:
json.dump(downloaded_files, file, ensure_ascii=False, indent=1)
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(manual()) asyncio.run(manual())
#asyncio.run(convert_files()) asyncio.run(convert_files())