From b5d63d1d00b9776a0f5d4d0e26acf0e6b72c8bd2 Mon Sep 17 00:00:00 2001 From: katboi01 Date: Sat, 15 Mar 2025 20:02:36 +0100 Subject: [PATCH] moved archive to a separate repo --- app.py | 2 - modules/Archive/database.py | 142 --------------- modules/Archive/databaseController.py | 48 ----- modules/Archive/endpoints/account_stats.py | 13 -- modules/Archive/endpoints/command.py | 23 --- modules/Archive/endpoints/commands.py | 24 --- modules/Archive/endpoints/new_query.py | 17 -- modules/Archive/endpoints/post.py | 15 -- modules/Archive/endpoints/posts.py | 55 ------ modules/Archive/endpoints/posts_count.py | 42 ----- modules/Archive/endpoints/query.py | 18 -- modules/Archive/endpoints/set_action.py | 35 ---- modules/Archive/load_from_file.py | 8 - modules/KF3/downloader.py | 193 ++++++++++++++++----- 14 files changed, 146 insertions(+), 489 deletions(-) delete mode 100644 modules/Archive/database.py delete mode 100644 modules/Archive/databaseController.py delete mode 100644 modules/Archive/endpoints/account_stats.py delete mode 100644 modules/Archive/endpoints/command.py delete mode 100644 modules/Archive/endpoints/commands.py delete mode 100644 modules/Archive/endpoints/new_query.py delete mode 100644 modules/Archive/endpoints/post.py delete mode 100644 modules/Archive/endpoints/posts.py delete mode 100644 modules/Archive/endpoints/posts_count.py delete mode 100644 modules/Archive/endpoints/query.py delete mode 100644 modules/Archive/endpoints/set_action.py delete mode 100644 modules/Archive/load_from_file.py diff --git a/app.py b/app.py index 902cac7..941405e 100644 --- a/app.py +++ b/app.py @@ -3,7 +3,6 @@ from flask_restful import Api from modules.KF3.database import Database as KF3DB from modules.Kingdom.database import Database as KFKDB -from modules.Archive.database import Database as KFADB from modules.proxy import AgreementProxy, LaunchProxy, UpdateProxy app = Flask(__name__) @@ -12,7 +11,6 @@ app.databases = {} api = Api(app) -KFADB(api) api.add_resource(LaunchProxy, "/DMM/launch") api.add_resource(UpdateProxy, "/DMM/filelist", "/DMM/update") api.add_resource(AgreementProxy, "/DMM/agreement") diff --git a/modules/Archive/database.py b/modules/Archive/database.py deleted file mode 100644 index ae3b8c4..0000000 --- a/modules/Archive/database.py +++ /dev/null @@ -1,142 +0,0 @@ -import json -from flask import Flask - - -from .databaseController import DatabaseController -from .endpoints.query import Query -from .endpoints.command import Command -from .endpoints.commands import Commands -from .endpoints.post import GetPost -from .endpoints.posts import GetPosts -from .endpoints.new_query import NewQuery -from .endpoints.set_action import SetAction -from .endpoints.posts_count import GetPostsCount -from .endpoints.account_stats import AccountStats - -class Database: - db : DatabaseController = None - app : Flask = None - - def __init__(self, api) -> None: - self.app = api.app - - if "Archive" in self.app.databases: - del self.app.databases["Archive"] - - self.reload_data("/home/pi/python/Katbots/JapariArchive/database.db") - - self.app.databases["Archive"] = self - - api.add_resource(Query, "/Archive/Query") - api.add_resource(NewQuery, "/Archive/NewQuery") - api.add_resource(Command, "/Archive/Command") - api.add_resource(Commands, "/Archive/Commands") - api.add_resource(GetPost, "/Archive/GetPost/") - api.add_resource(GetPosts, "/Archive/GetPosts") - api.add_resource(GetPostsCount, "/Archive/GetPosts/Count") - api.add_resource(AccountStats, "/Archive/AccountStats") - api.add_resource(SetAction, "/Archive/SetAction") - - def get_accounts(self): - query = f''' - SELECT x_handle, id FROM accounts - ORDER BY x_handle ASC''' - - return self.db.run_query(query) - - def get_account_stats(self): - query = '''SELECT x_handle, - SUM(IIF(x_posts.action_taken = 0, 1, 0)) as undecided, - SUM(IIF(x_posts.action_taken = 1, 1, 0)) as approved, - SUM(IIF(x_posts.action_taken = 2, 1, 0)) as deleted, - SUM(IIF(x_posts.action_taken = 3, 1, 0)) as hidden, - SUM(IIF(x_post_details.id is NULL, 1, 0)) as invalid -FROM accounts - LEFT JOIN x_posts on accounts.id = x_posts.account_id - LEFT JOIN x_post_details on x_post_details.id = x_posts.id -WHERE accounts.download_mode != 4 -GROUP BY account_id -ORDER BY x_handle''' - return self.db.run_query(query) - - def get_post(self, id): - query = f'''SELECT x_posts.id, cast(x_posts.id as TEXT) as id_str, x_posts.error_id, x_posts.action_taken, x_posts.is_saved, x_post_details.text, x_post_details.files, x_post_details.date, accounts.x_handle, accounts.rating from x_posts - LEFT JOIN x_post_details - ON x_posts.id = x_post_details.id - LEFT JOIN accounts - ON x_posts.account_id = accounts.id WHERE x_posts.id = {id} - LIMIT 1''' - result = self.db.run_query(query) - if len(result) == 0: - return None - else: - #return most recent post - return result[-1] - - def build_where_query(self, artist, actions_taken = [0, 1, 2, 3], last_id = -1, include_ratings = ["SFW", "NSFW"]): - where_query = "WHERE x_post_details.id NOT NULL AND x_posts.error_id = 0" - if last_id != -1: - where_query += f" AND x_posts.id < {last_id}" - if actions_taken != [0, 1, 2, 3]: - where_query += " AND (" + " OR ".join([f"x_posts.action_taken = {action}" for action in actions_taken]) + ")" - if include_ratings != ["SFW", "NSFW", "NSFL"]: - where_query += " AND (" + " OR ".join([f'accounts.rating = "{rating}"' for rating in include_ratings]) + ")" - if artist is not None and artist != "": - where_query += f' AND accounts.x_handle = "{artist}"' - return where_query - - def get_posts_count(self, artist, actions_taken = [0, 1, 2, 3], last_id = -1, include_ratings = ["SFW", "NSFW"]): - where_query = self.build_where_query(artist, actions_taken, last_id, include_ratings) - - query = f''' - SELECT count(*) as count FROM x_posts - LEFT JOIN x_post_details - ON x_posts.id = x_post_details.id - LEFT JOIN accounts - ON x_posts.account_id = accounts.id - {where_query}''' - - result = self.db.run_query(query) - return result[0]["count"] - - def get_posts(self, num_posts, artist, actions_taken = [0, 1, 2, 3], last_id = -1, offset = 0, include_ratings = ["SFW", "NSFW"], order = "DESC"): - num_posts = max(1, min(num_posts, 100)) - - order_by = "RANDOM()" if order == "RAND" else "x_posts.id ASC" if order == "ASC" else "x_posts.id DESC" - - where_query = self.build_where_query(artist, actions_taken, last_id, include_ratings) - - query = f''' - SELECT x_posts.id, cast(x_posts.id as TEXT) as id_str, x_posts.action_taken, x_posts.is_saved, x_post_details.text, x_post_details.files, x_post_details.date, accounts.x_handle, accounts.rating FROM x_posts - LEFT JOIN x_post_details - ON x_posts.id = x_post_details.id - LEFT JOIN accounts - ON x_posts.account_id = accounts.id - {where_query} - ORDER BY {order_by} - LIMIT {num_posts} OFFSET {offset}''' - - return self.db.run_query(query) - - def wrap_query_response(self, result, mode = "json"): - if result is None: - response = self.app.response_class(status=400) - else: - if mode == "json": - response = self.app.response_class( - response=json.dumps(result, ensure_ascii=False, indent=1), - status=200, - mimetype='application/json' - ) - elif mode == "text": - response = self.app.response_class( - response=str(result), - status=200, - mimetype='text/plain' - ) - - response.headers.add("Access-Control-Allow-Origin", "*") - return response - - def reload_data(self, database_path): - self.db = DatabaseController(database_path) \ No newline at end of file diff --git a/modules/Archive/databaseController.py b/modules/Archive/databaseController.py deleted file mode 100644 index a0f0009..0000000 --- a/modules/Archive/databaseController.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -import sqlite3 - -TABLE_ACCOUNTS = "accounts" -TABLE_X = "x_posts" - -class DatabaseController: - def __init__(self, db_name): - self.conn = sqlite3.connect(db_name) - self.conn.row_factory = sqlite3.Row - self.cursor = self.conn.cursor() - - def run_query(self, query): - try: - self.cursor.execute(query) - results = [dict(result) for result in self.cursor.fetchall()] - return results - except Exception as e: - print(e) - return [] - - def run_command(self, command): - try: - self.cursor.execute(command) - result = self.cursor.rowcount - self.conn.commit() - return result > 0 - except Exception as e: - self.conn.rollback() - print(command, e) - return False - - def run_commands(self, commands): - last_command = "" - try: - for command in commands: - last_command = command - self.cursor.execute(command) - result = self.cursor.rowcount - self.conn.commit() - return result > 0 - except Exception as e: - self.conn.rollback() - print(last_command, e) - return False - - def close(self): - self.conn.close() \ No newline at end of file diff --git a/modules/Archive/endpoints/account_stats.py b/modules/Archive/endpoints/account_stats.py deleted file mode 100644 index b6aa1a4..0000000 --- a/modules/Archive/endpoints/account_stats.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import annotations -from flask_restful import Resource -from flask import current_app as app -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from modules.Archive.database import Database - -class AccountStats(Resource): - def get(self): - db : Database = app.databases["Archive"] - result = db.get_account_stats() - return db.wrap_query_response(result) \ No newline at end of file diff --git a/modules/Archive/endpoints/command.py b/modules/Archive/endpoints/command.py deleted file mode 100644 index 8594a22..0000000 --- a/modules/Archive/endpoints/command.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations -from flask_restful import Resource -from flask import current_app as app, request -from typing import TYPE_CHECKING -import hashlib - -if TYPE_CHECKING: - from modules.Archive.database import Database - -class Command(Resource): - def post(self): - db : Database = app.databases["Archive"] - auth = request.headers.get('auth') - if auth is not None: - hash_obj = hashlib.sha256(auth.encode('utf-8')) - if hash_obj.hexdigest() == "63a3b0dba950e1015a110486518e5ceff8cff415041aba3dedb8dc5aa3b3dd16": - query = request.data.decode("utf-8") - result = db.db.run_command(query) - else: - result = None - else: - result = None - return db.wrap_query_response(result, mode="text") \ No newline at end of file diff --git a/modules/Archive/endpoints/commands.py b/modules/Archive/endpoints/commands.py deleted file mode 100644 index f644ea5..0000000 --- a/modules/Archive/endpoints/commands.py +++ /dev/null @@ -1,24 +0,0 @@ -from __future__ import annotations -import json -from flask_restful import Resource -from flask import current_app as app, request -from typing import TYPE_CHECKING -import hashlib - -if TYPE_CHECKING: - from modules.Archive.database import Database - -class Commands(Resource): - def post(self): - db : Database = app.databases["Archive"] - auth = request.headers.get('auth') - if auth is not None: - hash_obj = hashlib.sha256(auth.encode('utf-8')) - if hash_obj.hexdigest() == "63a3b0dba950e1015a110486518e5ceff8cff415041aba3dedb8dc5aa3b3dd16": - data = json.loads(request.data) - result = db.db.run_commands(data) - else: - result = None - else: - result = None - return db.wrap_query_response(result, mode="text") \ No newline at end of file diff --git a/modules/Archive/endpoints/new_query.py b/modules/Archive/endpoints/new_query.py deleted file mode 100644 index 37d540c..0000000 --- a/modules/Archive/endpoints/new_query.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import annotations -from flask_restful import Resource -from flask import current_app as app, request -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from modules.Archive.database import Database - -class NewQuery(Resource): - def post(self): - query = request.data.decode("utf-8") - - db : Database = app.databases["Archive"] - - result = db.db.run_query(query) - - return db.wrap_query_response(result) \ No newline at end of file diff --git a/modules/Archive/endpoints/post.py b/modules/Archive/endpoints/post.py deleted file mode 100644 index d83d432..0000000 --- a/modules/Archive/endpoints/post.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import annotations -from flask_restful import Resource -from flask import current_app as app -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from modules.Archive.database import Database - -class GetPost(Resource): - def get(self, id): - id = int(id) - db : Database = app.databases["Archive"] - - result = db.get_post(id) - return db.wrap_query_response(result) \ No newline at end of file diff --git a/modules/Archive/endpoints/posts.py b/modules/Archive/endpoints/posts.py deleted file mode 100644 index 05d235c..0000000 --- a/modules/Archive/endpoints/posts.py +++ /dev/null @@ -1,55 +0,0 @@ -from __future__ import annotations -from flask_restful import Resource -from flask import current_app as app, request -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from modules.Archive.database import Database - -class GetPosts(Resource): - def get(self): - db : Database = app.databases["Archive"] - try: - count = int(request.args["count"]) if "count" in request.args else 20 - page = int(request.args["page"]) if "page" in request.args else 1 - artist = request.args["artist"] if "artist" in request.args else None - last_id = int(request.args["last_id"]) if "last_id" in request.args else -1 - except: - response = app.response_class(status=400) - response.headers.add("Access-Control-Allow-Origin", "*") - return response - - actions = [] - if "undecided" in request.args: - actions.append(0) - if "approved" in request.args: - actions.append(1) - if "denied" in request.args: - actions.append(2) - if "hidden" in request.args: - actions.append(3) - if actions == []: - actions = [0,1,2,3] - - ratings = [] - if "SFW" in request.args: - ratings.append("SFW") - if "NSFW" in request.args: - ratings.append("NSFW") - if "NSFL" in request.args: - ratings.append("NSFL") - if ratings == []: - ratings = ["SFW", "NSFW"] - - if "random" in request.args: - order = "RAND" - elif "ascending" in request.args: - order = "ASC" - else: - order = "DESC" - - real_page = page-1 - offset = real_page * count - - result = db.get_posts(count, artist, actions_taken=actions, last_id= -1, offset= offset, include_ratings= ratings, order=order) - return db.wrap_query_response(result) \ No newline at end of file diff --git a/modules/Archive/endpoints/posts_count.py b/modules/Archive/endpoints/posts_count.py deleted file mode 100644 index 68712f4..0000000 --- a/modules/Archive/endpoints/posts_count.py +++ /dev/null @@ -1,42 +0,0 @@ -from __future__ import annotations -from flask_restful import Resource -from flask import current_app as app, request -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from modules.Archive.database import Database - -class GetPostsCount(Resource): - def get(self): - db : Database = app.databases["Archive"] - try: - artist = request.args["artist"] if "artist" in request.args else None - except: - response = app.response_class(status=400) - response.headers.add("Access-Control-Allow-Origin", "*") - return response - - actions = [] - if "undecided" in request.args: - actions.append(0) - if "approved" in request.args: - actions.append(1) - if "denied" in request.args: - actions.append(2) - if "hidden" in request.args: - actions.append(3) - if actions == []: - actions = [0,1,2,3] - - ratings = [] - if "SFW" in request.args: - ratings.append("SFW") - if "NSFW" in request.args: - ratings.append("NSFW") - if "NSFL" in request.args: - ratings.append("NSFL") - if ratings == []: - ratings = ["SFW", "NSFW"] - - result = db.get_posts_count(artist, actions_taken=actions, last_id= -1, include_ratings= ratings) - return db.wrap_query_response(result, mode="text") \ No newline at end of file diff --git a/modules/Archive/endpoints/query.py b/modules/Archive/endpoints/query.py deleted file mode 100644 index 774ef4d..0000000 --- a/modules/Archive/endpoints/query.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import annotations -from flask_restful import Resource -from flask import current_app as app, request -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from modules.Archive.database import Database - -class Query(Resource): - def post(self): - query = request.data.decode("utf-8") - - db : Database = app.databases["Archive"] - - result = db.db.run_query(query) - result = [list(d.values()) for d in result] - - return db.wrap_query_response(result) \ No newline at end of file diff --git a/modules/Archive/endpoints/set_action.py b/modules/Archive/endpoints/set_action.py deleted file mode 100644 index eb6da8c..0000000 --- a/modules/Archive/endpoints/set_action.py +++ /dev/null @@ -1,35 +0,0 @@ -from __future__ import annotations -import json -from flask_restful import Resource -from flask import current_app as app, request -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from modules.Archive.database import Database - -class SetAction(Resource): - def post(self): - data = json.loads(request.data) - - try: - bypass = ("allow_override" in data) and data["allow_override"] - action = data["action_taken"] - if "id_str" in data: - id = int(data["id_str"]) - elif "id" in data: - id = data["id"] - else: - raise Exception("no id (int) or id_str (str) specified") - except Exception as e: - print(e) - response = app.response_class(response=e, status=400) - response.headers.add("Access-Control-Allow-Origin", "*") - return response - - db : Database = app.databases["Archive"] - query = f"UPDATE x_posts SET action_taken = {action} WHERE id = {id}" - if not bypass: - query += " AND action_taken = 0" - - result = db.db.run_command(query) - return db.wrap_query_response(result, mode="text") \ No newline at end of file diff --git a/modules/Archive/load_from_file.py b/modules/Archive/load_from_file.py deleted file mode 100644 index 6176829..0000000 --- a/modules/Archive/load_from_file.py +++ /dev/null @@ -1,8 +0,0 @@ -import sys -from databaseController import DatabaseController - -db = DatabaseController("/home/pi/python/Katbots/JapariArchive/database.db") -with open(sys.argv[1], "rt") as file: - lines = [line for line in file.readlines() if line.strip()] - -print(db.run_commands(lines)) diff --git a/modules/KF3/downloader.py b/modules/KF3/downloader.py index 1951f65..a2b3b9f 100644 --- a/modules/KF3/downloader.py +++ b/modules/KF3/downloader.py @@ -9,6 +9,7 @@ import platform import aiohttp from UnityPy import enums +import requests from ..Shared.utility import divide_chunks from ..Shared.downloading import download_bytes, download_text @@ -41,7 +42,7 @@ def encode(text): text[i] = text[i] ^ key[i % (len(key))] return hashed[:4].hex() + text.hex() + checksum.hex() -async def download_cache(server_name, server : str): +async def download_cache(server_name, server_app : str, server_cdn : str = None): session = aiohttp.ClientSession() downloaded_files = [] @@ -60,7 +61,7 @@ async def download_cache(server_name, server : str): old_mst_ver = dict([(entry["type"], entry["version"]) for entry in json.load(file)]) param = encode(json.dumps({'dmm_viewer_id':0})) - request = await download_bytes(server + "paradesv/common/MstVersion.do?param=" + param, session) + request = await download_bytes(server_app + "paradesv/common/MstVersion.do?param=" + param, session) result = gzip.decompress(request) new_mst = json.loads(result) new_mst_ver = dict([(entry["type"], entry["version"]) for entry in new_mst["mst_ver"]]) @@ -74,7 +75,7 @@ async def download_cache(server_name, server : str): downloaded_files.append(key) param = encode(json.dumps({'type':key, 'dmm_viewer_id':0})) - request = await download_bytes(server + "paradesv/common/MstData.do?param=" + param, session) + request = await download_bytes(server_app + "paradesv/common/MstData.do?param=" + param, session) result = gzip.decompress(request) response = json.loads(result) data = base64.b64decode(response["data"]) @@ -83,8 +84,14 @@ async def download_cache(server_name, server : str): with open(file_path_json, "wt", encoding="utf-8") as out_file: data = gzip.decompress(data) data = json.loads(data) - # if key == "GACHA_DATA": - # download_banners(data, server_name) + if key == "GACHA_DATA": + await download_banners(data, server_name, server_app, server_cdn, session) + if key == "BANNER_DATA": + await download_banner_data(data, server_name, server_cdn, session) + if key == "EVENT_BANNER_DATA": + await download_event_banner_data(data, server_name, server_cdn, session) + if key == "EVENT_DATA": + await download_event_data(data, server_name, server_cdn, session) json.dump(data, out_file, ensure_ascii=False, indent=1) old_mst_ver[key] = new_mst_ver[key] @@ -96,21 +103,21 @@ async def download_cache(server_name, server : str): await session.close() return downloaded_files -async def download_banners(gacha_data, server_name, session): - path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/banners/" +async def download_banners(gacha_data, server_name, server_app, server_cdn, session): + path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\banners\\" os.makedirs(path, exist_ok=True) - for entry in gacha_data: + async def download_banner(entry): banner_name = entry["banner"] if banner_name == "" or banner_name == None: - continue + return banner_name += ".png" file_path = path + banner_name if os.path.exists(file_path): - continue + return - file_url = "https://parade-mobile-prod-cdn.kemono-friends-3.jp/Texture2D/GachaTop/" + banner_name - file_url_alt = "https://parade-mobile-develop01-app.kemono-friends-3.jp/Texture2D/GachaTop/" + banner_name + file_url_alt = f"{server_cdn}Texture2D/GachaTop/{banner_name}" + file_url = f"{server_app}Texture2D/GachaTop/{banner_name}" status = 0 async with session.get(file_url) as resp: @@ -120,11 +127,118 @@ async def download_banners(gacha_data, server_name, session): async with session.get(file_url_alt) as resp: response = await resp.read() status = resp.status - if status != 200: continue + if status != 200: return with open(file_path, "wb") as file: file.write(response) + + chunked_files_to_download = list(divide_chunks(gacha_data, 20)) + + for chunk in chunked_files_to_download: + tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk] + await asyncio.wait(tasks) + +async def download_banner_data(gacha_data, server_name, server_cdn, session : aiohttp.ClientSession): + path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\" + + async def download_banner(entry): + name = entry["bannerName"] + if name == "" or name == None: + return + bannerImagePath = "Texture2D/HomeBanner/home_banner_" + name + ".png" + bannerImagePathByQuestTop = "Texture2D/QuestTop/questtop_banner_" + name + ".png" + bannerImagePathEvent = "Texture2D/EventTop/eventtop_banner_" + name + ".png" + banners = [bannerImagePath, bannerImagePathByQuestTop, bannerImagePathEvent] + + for banner in banners: + file_path = path + banner + if os.path.exists(file_path): + continue + + fileDir = "/".join(file_path.split("/")[:-1]) + os.makedirs(fileDir, exist_ok=True) + file_url = f"{server_cdn}{banner}" + + try: + with open(file_path, 'wb') as out_file: + req = requests.get(file_url, stream=True) + if req.status_code != 200: raise Exception("lol") + out_file.write(req.content) + except Exception as ex: + os.remove(file_path) + print(ex) + continue + + chunked_files_to_download = list(divide_chunks(gacha_data, 20)) + + for chunk in chunked_files_to_download: + tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk] + await asyncio.wait(tasks) + +async def download_event_banner_data(gacha_data, server_name, server_cdn, session : aiohttp.ClientSession): + path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\" + + async def download_banner(entry): + name = entry["bannerName"] + if name == "" or name == None: + return + banner = "Texture2D/HomeBigBanner/home_bigbanner_" + name + ".png" + + file_path = path + banner + if os.path.exists(file_path): + return + fileDir = "/".join(file_path.split("/")[:-1]) + os.makedirs(fileDir, exist_ok=True) + file_url = f"{server_cdn}{banner}" + + try: + with open(file_path, 'wb') as out_file: + req = requests.get(file_url, stream=True) + if req.status_code != 200: raise Exception("lol") + out_file.write(req.content) + except Exception as ex: + os.remove(file_path) + print(ex) + + chunked_files_to_download = list(divide_chunks(gacha_data, 20)) + + for chunk in chunked_files_to_download: + tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk] + await asyncio.wait(tasks) + + +async def download_event_data(gacha_data, server_name, server_cdn, session : aiohttp.ClientSession): + path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\" + + async def download_banner(entry): + name = entry["missionBannerFilename"] + if name == "" or name == None: + return + banner = "Texture2D/Mission/" + name + ".png" + + file_path = path + banner + if os.path.exists(file_path): + return + + fileDir = "/".join(file_path.split("/")[:-1]) + os.makedirs(fileDir, exist_ok=True) + file_url = f"{server_cdn}{banner}" + + try: + with open(file_path, 'wb') as out_file: + req = requests.get(file_url, stream=True) + if req.status_code != 200: raise Exception("lol") + out_file.write(req.content) + except Exception as ex: + os.remove(file_path) + print(ex) + + chunked_files_to_download = list(divide_chunks(gacha_data, 20)) + + for chunk in chunked_files_to_download: + tasks = [asyncio.create_task(download_banner(banner)) for banner in chunk] + await asyncio.wait(tasks) async def download_files(server_name, asset_bundle_url, srv_platform : str): def parse_ab_list(filecontent : str): @@ -153,7 +267,11 @@ async def download_files(server_name, asset_bundle_url, srv_platform : str): extract(data, extract_path) session = aiohttp.ClientSession() - path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/assets/{srv_platform}/" + + if platform.system() == "Windows": + path = f"D:\\Codebase\\KFKDecrypt\\assets\\KF3\\{server_name}\\assets\\{srv_platform}\\" + else: + path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/assets/{srv_platform}/" os.makedirs(path, exist_ok=True) @@ -190,7 +308,7 @@ async def download_files(server_name, asset_bundle_url, srv_platform : str): files_to_download.append(key) - chunked_files_to_download = list(divide_chunks(files_to_download, 5)) + chunked_files_to_download = list(divide_chunks(files_to_download, 10)) for chunk in chunked_files_to_download: tasks = [asyncio.create_task(download_file(url_assets, file, path, session)) for file in chunk] @@ -227,49 +345,30 @@ async def convert_files(): print("Conversion failed", f) async def manual(): - - # session = aiohttp.ClientSession() - - # path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/cache/" - - # await session.close() - - # return - downloaded_cache = {} downloaded_files = {} + + server_cdn = "https://kf3-prod-cdn.kf3.wayi.com.tw/" + server_app = "https://kf3-prod-app.kf3.wayi.com.tw/" async with aiohttp.ClientSession() as session: param = encode(json.dumps({"version":"1.0.0","dmm_viewer_id":0,"platform":1})) - request = await download_bytes(servers[0] + "paradesv/common/GetUrl.do?param=" + param, session) + request = await download_bytes(server_app + "paradesv/common/GetUrl.do?param=" + param, session) result = gzip.decompress(request) response = json.loads(result) asset_bundle_url = response["asset_bundle_url"] urlName = asset_bundle_url.split("-")[2] - print("downloading from", servers[0]) - downloaded_cache = await download_cache(urlName, servers[0]) - downloaded_files = await download_files(urlName, asset_bundle_url, "Windows") + print("downloading from", server_app) + downloaded_cache = await download_cache("TW", server_app, server_cdn) + downloaded_files = await download_files("TW", asset_bundle_url, "Android") - if downloaded_cache != [] and downloaded_cache != None: - with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_cache.json", "wt", encoding="utf-8") as file: - json.dump(downloaded_cache, file, ensure_ascii=False, indent=1) - if downloaded_files != [] and downloaded_files != None: - with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_files.json", "wt", encoding="utf-8") as file: - json.dump(downloaded_files, file, ensure_ascii=False, indent=1) - - print("downloading from", servers[1]) - asset_bundle_url = "https://parade-mobile-develop01-app.kemono-friends-3.jp/AssetBundles/0.0.0/latest" - urlName = asset_bundle_url.split("-")[2] - downloaded_cache = await download_cache(urlName, servers[1]) - downloaded_files = await download_files(urlName, asset_bundle_url, "Windows") - - if downloaded_cache != [] and downloaded_cache != None: - with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_dev_cache.json", "wt", encoding="utf-8") as file: - json.dump(downloaded_cache, file, ensure_ascii=False, indent=1) - if downloaded_files != [] and downloaded_files != None: - with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_dev_files.json", "wt", encoding="utf-8") as file: - json.dump(downloaded_files, file, ensure_ascii=False, indent=1) + # if downloaded_cache != [] and downloaded_cache != None: + # with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_cache.json", "wt", encoding="utf-8") as file: + # json.dump(downloaded_cache, file, ensure_ascii=False, indent=1) + # if downloaded_files != [] and downloaded_files != None: + # with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_files.json", "wt", encoding="utf-8") as file: + # json.dump(downloaded_files, file, ensure_ascii=False, indent=1) if __name__ == "__main__": asyncio.run(manual())