removed ArchiveOld, added more data to posts, added DMM proxy endpoint

app.py

@@ -3,6 +3,8 @@ from flask_restful import Api
 
 from modules.KF3.database import Database as KF3DB
 from modules.Kingdom.database import Database as KFKDB
+from modules.Archive.database import Database as KFADB
+from modules.proxy import AgreementProxy, LaunchProxy, UpdateProxy
 
 app = Flask(__name__)
 app.config['JSON_AS_ASCII'] = False
@@ -10,6 +12,10 @@ app.databases = {}
 
 api = Api(app)
 
+KFADB(api)
+api.add_resource(LaunchProxy, "/DMM/launch")
+api.add_resource(UpdateProxy, "/DMM/filelist", "/DMM/update")
+api.add_resource(AgreementProxy, "/DMM/agreement")
 KF3DB(api)
 KFKDB(api)
 

modules/Archive/database.py

@@ -0,0 +1,142 @@
+import json
+from flask import Flask
+
+
+from .databaseController import DatabaseController
+from .endpoints.query import Query
+from .endpoints.command import Command
+from .endpoints.commands import Commands
+from .endpoints.post import GetPost
+from .endpoints.posts import GetPosts
+from .endpoints.new_query import NewQuery
+from .endpoints.set_action import SetAction
+from .endpoints.posts_count import GetPostsCount
+from .endpoints.account_stats import AccountStats
+
+class Database:
+    db : DatabaseController = None
+    app : Flask = None
+
+    def __init__(self, api) -> None:
+        self.app = api.app
+
+        if "Archive" in self.app.databases:
+            del self.app.databases["Archive"]
+
+        self.reload_data("/home/pi/python/Katbots/JapariArchive/database.db")
+
+        self.app.databases["Archive"] = self
+
+        api.add_resource(Query, "/Archive/Query")
+        api.add_resource(NewQuery, "/Archive/NewQuery")
+        api.add_resource(Command, "/Archive/Command")
+        api.add_resource(Commands, "/Archive/Commands")
+        api.add_resource(GetPost, "/Archive/GetPost/<id>")
+        api.add_resource(GetPosts, "/Archive/GetPosts")
+        api.add_resource(GetPostsCount, "/Archive/GetPosts/Count")
+        api.add_resource(AccountStats, "/Archive/AccountStats")
+        api.add_resource(SetAction, "/Archive/SetAction")
+  
+    def get_accounts(self):
+        query = f'''
+        SELECT x_handle, id FROM accounts 
+        ORDER BY x_handle ASC'''
+
+        return self.db.run_query(query)
+
+    def get_account_stats(self):
+        query = '''SELECT x_handle,
+	SUM(IIF(x_posts.action_taken = 0, 1, 0)) as undecided,
+	SUM(IIF(x_posts.action_taken = 1, 1, 0)) as approved,
+	SUM(IIF(x_posts.action_taken = 2, 1, 0)) as deleted,
+	SUM(IIF(x_posts.action_taken = 3, 1, 0)) as hidden,
+	SUM(IIF(x_post_details.id is NULL, 1, 0)) as invalid
+FROM accounts 
+	LEFT JOIN x_posts on accounts.id = x_posts.account_id 
+	LEFT JOIN x_post_details on x_post_details.id = x_posts.id
+WHERE accounts.download_mode != 4
+GROUP BY account_id 
+ORDER BY x_handle'''
+        return self.db.run_query(query)
+
+    def get_post(self, id):
+        query = f'''SELECT x_posts.id, cast(x_posts.id as TEXT) as id_str, x_posts.error_id, x_posts.action_taken, x_posts.is_saved, x_post_details.text, x_post_details.files, x_post_details.date, accounts.x_handle, accounts.rating from x_posts
+        LEFT JOIN x_post_details
+            ON x_posts.id = x_post_details.id
+        LEFT JOIN accounts
+            ON x_posts.account_id = accounts.id WHERE x_posts.id = {id}
+        LIMIT 1'''
+        result = self.db.run_query(query)
+        if len(result) == 0:
+            return None
+        else:
+            #return most recent post
+            return result[-1]
+
+    def build_where_query(self, artist, actions_taken = [0, 1, 2, 3], last_id = -1, include_ratings = ["SFW", "NSFW"]):
+        where_query = "WHERE x_post_details.id NOT NULL AND x_posts.error_id = 0"
+        if last_id != -1:
+            where_query += f" AND x_posts.id < {last_id}"
+        if actions_taken != [0, 1, 2, 3]:
+            where_query += " AND (" + " OR ".join([f"x_posts.action_taken = {action}" for action in actions_taken]) + ")"
+        if include_ratings != ["SFW", "NSFW", "NSFL"]:
+            where_query += " AND (" + " OR ".join([f'accounts.rating = "{rating}"' for rating in include_ratings]) + ")"
+        if artist is not None and artist != "":
+            where_query += f' AND accounts.x_handle = "{artist}"'
+        return where_query
+
+    def get_posts_count(self, artist, actions_taken = [0, 1, 2, 3], last_id = -1, include_ratings = ["SFW", "NSFW"]):
+        where_query = self.build_where_query(artist, actions_taken, last_id, include_ratings)
+
+        query = f'''
+        SELECT count(*) as count FROM x_posts 
+        LEFT JOIN x_post_details
+            ON x_posts.id = x_post_details.id
+        LEFT JOIN accounts
+            ON x_posts.account_id = accounts.id
+        {where_query}'''
+
+        result = self.db.run_query(query)
+        return result[0]["count"]
+
+    def get_posts(self, num_posts, artist, actions_taken = [0, 1, 2, 3], last_id = -1, offset = 0, include_ratings = ["SFW", "NSFW"], order = "DESC"):
+        num_posts = max(1, min(num_posts, 100))
+
+        order_by = "RANDOM()" if order == "RAND" else "x_posts.id ASC" if order == "ASC" else "x_posts.id DESC"
+
+        where_query = self.build_where_query(artist, actions_taken, last_id, include_ratings)
+
+        query = f'''
+        SELECT x_posts.id, cast(x_posts.id as TEXT) as id_str, x_posts.action_taken, x_posts.is_saved, x_post_details.text, x_post_details.files, x_post_details.date, accounts.x_handle, accounts.rating FROM x_posts 
+        LEFT JOIN x_post_details
+            ON x_posts.id = x_post_details.id
+        LEFT JOIN accounts
+            ON x_posts.account_id = accounts.id
+        {where_query}
+        ORDER BY {order_by}
+        LIMIT {num_posts} OFFSET {offset}'''
+
+        return self.db.run_query(query)
+
+    def wrap_query_response(self, result, mode = "json"):
+        if result is None:
+            response = self.app.response_class(status=400)
+        else:
+            if mode == "json":
+                response = self.app.response_class(
+                    response=json.dumps(result, ensure_ascii=False, indent=1),
+                    status=200,
+                    mimetype='application/json'
+            )
+            elif mode == "text":
+                response = self.app.response_class(
+                    response=str(result),
+                    status=200,
+                    mimetype='text/plain'
+            )
+                
+        response.headers.add("Access-Control-Allow-Origin", "*")
+        return response
+
+    def reload_data(self, database_path):
+        self.db = DatabaseController(database_path)

modules/Archive/databaseController.py

@@ -0,0 +1,48 @@
+import os
+import sqlite3
+
+TABLE_ACCOUNTS  = "accounts"
+TABLE_X         = "x_posts"
+
+class DatabaseController:
+    def __init__(self, db_name):
+        self.conn = sqlite3.connect(db_name)
+        self.conn.row_factory = sqlite3.Row
+        self.cursor = self.conn.cursor()
+
+    def run_query(self, query):
+        try:
+            self.cursor.execute(query)
+            results = [dict(result) for result in self.cursor.fetchall()]
+            return results
+        except Exception as e:
+            print(e)
+            return []
+        
+    def run_command(self, command):
+        try:
+            self.cursor.execute(command)
+            result = self.cursor.rowcount
+            self.conn.commit()
+            return result > 0
+        except Exception as e:
+            self.conn.rollback()
+            print(command, e)
+            return False
+
+    def run_commands(self, commands):
+        last_command = ""
+        try:
+            for command in commands:
+                last_command = command
+                self.cursor.execute(command)
+            result = self.cursor.rowcount
+            self.conn.commit()
+            return result > 0
+        except Exception as e:
+            self.conn.rollback()
+            print(last_command, e)
+            return False
+        
+    def close(self):
+        self.conn.close()

modules/Archive/endpoints/account_stats.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+from flask_restful import Resource
+from flask import current_app as app
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from modules.Archive.database import Database
+
+class AccountStats(Resource):
+    def get(self):
+        db : Database = app.databases["Archive"]
+        result = db.get_account_stats()
+        return db.wrap_query_response(result)

modules/Archive/endpoints/command.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+from flask_restful import Resource
+from flask import current_app as app, request
+from typing import TYPE_CHECKING
+import hashlib
+
+if TYPE_CHECKING:
+    from modules.Archive.database import Database
+
+class Command(Resource):
+    def post(self):
+        db : Database = app.databases["Archive"]
+        auth = request.headers.get('auth')
+        if auth is not None:
+            hash_obj = hashlib.sha256(auth.encode('utf-8'))
+            if hash_obj.hexdigest() == "63a3b0dba950e1015a110486518e5ceff8cff415041aba3dedb8dc5aa3b3dd16":
+                query = request.data.decode("utf-8")
+                result = db.db.run_command(query)
+            else: 
+                result = None
+        else:
+            result = None
+        return db.wrap_query_response(result, mode="text")

modules/Archive/endpoints/commands.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+import json
+from flask_restful import Resource
+from flask import current_app as app, request
+from typing import TYPE_CHECKING
+import hashlib
+
+if TYPE_CHECKING:
+    from modules.Archive.database import Database
+
+class Commands(Resource):
+    def post(self):
+        db : Database = app.databases["Archive"]
+        auth = request.headers.get('auth')
+        if auth is not None:
+            hash_obj = hashlib.sha256(auth.encode('utf-8'))
+            if hash_obj.hexdigest() == "63a3b0dba950e1015a110486518e5ceff8cff415041aba3dedb8dc5aa3b3dd16":
+                data = json.loads(request.data)
+                result = db.db.run_commands(data)
+            else: 
+                result = None
+        else:
+            result = None
+        return db.wrap_query_response(result, mode="text")

modules/Archive/endpoints/new_query.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+from flask_restful import Resource
+from flask import current_app as app, request
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from modules.Archive.database import Database
+
+class NewQuery(Resource):
+    def post(self):
+        query = request.data.decode("utf-8")
+        
+        db : Database = app.databases["Archive"]
+
+        result = db.db.run_query(query)
+
+        return db.wrap_query_response(result)

modules/Archive/endpoints/post.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+from flask_restful import Resource
+from flask import current_app as app
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from modules.Archive.database import Database
+
+class GetPost(Resource):
+    def get(self, id):
+        id = int(id)
+        db : Database = app.databases["Archive"]
+
+        result = db.get_post(id)
+        return db.wrap_query_response(result) 

modules/Archive/endpoints/posts.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+from flask_restful import Resource
+from flask import current_app as app, request
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from modules.Archive.database import Database
+
+class GetPosts(Resource):
+    def get(self):
+        db : Database = app.databases["Archive"]
+        try:
+            count = int(request.args["count"]) if "count" in request.args else 20
+            page = int(request.args["page"]) if "page" in request.args else 1
+            artist = request.args["artist"] if "artist" in request.args else None
+            last_id = int(request.args["last_id"]) if "last_id" in request.args else -1
+        except:
+            response = app.response_class(status=400)
+            response.headers.add("Access-Control-Allow-Origin", "*")
+            return response   
+
+        actions = []
+        if "undecided" in request.args:
+            actions.append(0)
+        if "approved" in request.args:
+            actions.append(1)
+        if "denied" in request.args:
+            actions.append(2)
+        if "hidden" in request.args:
+            actions.append(3)
+        if actions == []:
+            actions = [0,1,2,3]
+
+        ratings = []
+        if "SFW" in request.args:
+            ratings.append("SFW")
+        if "NSFW" in request.args:
+            ratings.append("NSFW")
+        if "NSFL" in request.args:
+            ratings.append("NSFL")
+        if ratings == []:
+            ratings = ["SFW", "NSFW"]
+
+        if "random" in request.args:
+            order = "RAND"
+        elif "ascending" in request.args:
+            order = "ASC"
+        else:
+            order = "DESC"
+
+        real_page = page-1
+        offset = real_page * count
+
+        result = db.get_posts(count, artist, actions_taken=actions, last_id= -1, offset= offset, include_ratings= ratings, order=order)
+        return db.wrap_query_response(result)

modules/Archive/endpoints/posts_count.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+from flask_restful import Resource
+from flask import current_app as app, request
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from modules.Archive.database import Database
+
+class GetPostsCount(Resource):
+    def get(self):
+        db : Database = app.databases["Archive"]
+        try:
+            artist = request.args["artist"] if "artist" in request.args else None
+        except:
+            response = app.response_class(status=400)
+            response.headers.add("Access-Control-Allow-Origin", "*")
+            return response   
+
+        actions = []
+        if "undecided" in request.args:
+            actions.append(0)
+        if "approved" in request.args:
+            actions.append(1)
+        if "denied" in request.args:
+            actions.append(2)
+        if "hidden" in request.args:
+            actions.append(3)
+        if actions == []:
+            actions = [0,1,2,3]
+
+        ratings = []
+        if "SFW" in request.args:
+            ratings.append("SFW")
+        if "NSFW" in request.args:
+            ratings.append("NSFW")
+        if "NSFL" in request.args:
+            ratings.append("NSFL")
+        if ratings == []:
+            ratings = ["SFW", "NSFW"]
+
+        result = db.get_posts_count(artist, actions_taken=actions, last_id= -1, include_ratings= ratings)
+        return db.wrap_query_response(result, mode="text") 

modules/Archive/endpoints/query.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+from flask_restful import Resource
+from flask import current_app as app, request
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from modules.Archive.database import Database
+
+class Query(Resource):
+    def post(self):
+        query = request.data.decode("utf-8")
+        
+        db : Database = app.databases["Archive"]
+
+        result = db.db.run_query(query)
+        result = [list(d.values()) for d in result]
+
+        return db.wrap_query_response(result)

modules/Archive/endpoints/set_action.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+import json
+from flask_restful import Resource
+from flask import current_app as app, request
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from modules.Archive.database import Database
+
+class SetAction(Resource):
+    def post(self):
+        data = json.loads(request.data)
+
+        try:
+            bypass = ("allow_override" in data) and data["allow_override"]
+            action = data["action_taken"]
+            if "id_str" in data:
+                id = int(data["id_str"])
+            elif "id" in data:
+                id = data["id"]
+            else:
+                raise Exception("no id (int) or id_str (str) specified")
+        except Exception as e:
+            print(e)
+            response = app.response_class(response=e, status=400)
+            response.headers.add("Access-Control-Allow-Origin", "*")
+            return response
+
+        db : Database = app.databases["Archive"]
+        query = f"UPDATE x_posts SET action_taken = {action} WHERE id = {id}"
+        if not bypass:
+            query += " AND action_taken = 0"
+
+        result = db.db.run_command(query)
+        return db.wrap_query_response(result, mode="text")

modules/Archive/load_from_file.py

@@ -0,0 +1,8 @@
+import sys
+from databaseController import DatabaseController
+
+db = DatabaseController("/home/pi/python/Katbots/JapariArchive/database.db")
+with open(sys.argv[1], "rt") as file:
+    lines = [line for line in file.readlines() if line.strip()]
+
+print(db.run_commands(lines))

modules/KF3/database.py

@@ -123,7 +123,7 @@ class Database:
         chara = {}
         
         charaData = self.charaData[id]
-        alphaBase = self.paramAlphaBases[id]
+        alphaBase = self.paramAlphaBases[id] if id in self.paramAlphaBases else None
 
         wrLocked = False
         promoIds = []
@@ -190,6 +190,12 @@ class Database:
         else:
             level_curve = self.limitlevel_rising_status[patternId] if patternId != 0 else None
 
+        if alphaBase is None:
+            chara["stats_min"] = {"level" : 0,"status" : 0,"wr" : 0,"hp" : 0,"atk" : 0,"def" : 0,"evd" : 0,"beat" : 0,"act" : 0,"try" : 0}
+            chara["stats_max"] = {"level" : 0,"status" : 0,"wr" : 0,"hp" : 0,"atk" : 0,"def" : 0,"evd" : 0,"beat" : 0,"act" : 0,"try" : 0}
+            chara["plasmPoint"] = 0
+            chara["cards"] = {0,0,0,0,0}
+        else:
             chara["stats_min"] = get_all_stats(chara, alphaBase, max_level = False, rising_status_pattern=level_curve)
             chara["stats_max"] = get_all_stats(chara, alphaBase, max_level = True, rising_status_pattern=level_curve)
             chara["plasmPoint"] = alphaBase["plasmPoint"]
@@ -200,6 +206,7 @@ class Database:
                 {"type":alphaBase["orderCardType03"], "value":alphaBase["orderCardValue03"]},
                 {"type":alphaBase["orderCardType04"], "value":alphaBase["orderCardValue04"]}
             ]
+
         chara["synergy_flag"] = self.paramArts[id]["authParam"]["SynergyFlag"] if id in self.paramArts else 0
         
         chara["arts"] = self.paramArts[id] if id in self.paramArts else None

modules/KF3/downloader.py

@@ -82,7 +82,10 @@ async def download_cache(server_name, server : str):
             file.write(data)
         with open(file_path_json, "wt", encoding="utf-8") as out_file:
             data = gzip.decompress(data)
-            json.dump(json.loads(data), out_file, ensure_ascii=False, indent=1)
+            data = json.loads(data)
+            # if key == "GACHA_DATA":
+            #     download_banners(data, server_name)
+            json.dump(data, out_file, ensure_ascii=False, indent=1)
 
         old_mst_ver[key] = new_mst_ver[key]
 
@@ -93,6 +96,36 @@ async def download_cache(server_name, server : str):
     await session.close()
     return downloaded_files
 
+async def download_banners(gacha_data, server_name, session):
+    path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/banners/"
+    os.makedirs(path, exist_ok=True)
+
+    for entry in gacha_data:
+        banner_name = entry["banner"]
+        if banner_name == "" or banner_name == None:
+            continue
+        banner_name += ".png"
+        file_path = path + banner_name
+        if os.path.exists(file_path):
+            continue
+        
+        file_url = "https://parade-mobile-prod-cdn.kemono-friends-3.jp/Texture2D/GachaTop/" + banner_name
+        file_url_alt = "https://parade-mobile-develop01-app.kemono-friends-3.jp/Texture2D/GachaTop/" + banner_name
+        
+        status = 0
+        async with session.get(file_url) as resp:
+            response = await resp.read()
+            status = resp.status
+        if status != 200:
+            async with session.get(file_url_alt) as resp:
+                response = await resp.read()
+                status = resp.status
+        if status != 200: continue
+
+        with open(file_path, "wb") as file:
+            file.write(response)
+        
+
 async def download_files(server_name, asset_bundle_url, srv_platform : str):
     def parse_ab_list(filecontent : str):
         out = {}
@@ -194,6 +227,15 @@ async def convert_files():
             print("Conversion failed", f)
 
 async def manual():
+
+    # session = aiohttp.ClientSession()
+
+    # path = f"/var/www/html/Katworks/KF/assets/KF3/{server_name}/cache/"
+
+    # await session.close()
+
+    # return
+
     downloaded_cache = {}
     downloaded_files = {}
     

modules/proxy.py

@@ -0,0 +1,48 @@
+from flask_restful import Resource
+from flask import current_app as app, request, Request
+
+import requests
+
+def relayRequest(user_request : Request, url, headers):
+    with requests.Session() as session:
+        requests.utils.add_dict_to_cookiejar(session.cookies, user_request.cookies)
+        response = session.post(url, headers=headers, data=request.data)
+        
+        result = app.response_class(
+                response=response.text,
+                status=200,
+                mimetype='application/json'
+        )
+        result.headers.add("Access-Control-Allow-Origin", "*")
+
+    return result
+
+class LaunchProxy(Resource):
+    def post(self):
+        url = "https://apidgp-gameplayer.games.dmm.com/v5/launch/cl"
+        headers = {"User-Agent": "DMMGamePlayer5-Win/5.3.12 Electron/32.1.0",
+                "Client-App": "DMMGamePlayer5",
+                    "Client-version": "5.3.12",
+                    "Content-Type": "application/json"}
+
+        return relayRequest(request, url, headers) 
+        
+class UpdateProxy(Resource):
+    def post(self):
+        url = "https://apidgp-gameplayer.games.dmm.com/v5/r2/launch/cl"
+        headers = {"User-Agent": "DMMGamePlayer5-Win/5.3.12 Electron/32.1.0",
+                "Client-App": "DMMGamePlayer5",
+                    "Client-version": "5.3.12",
+                    "Content-Type": "application/json"}
+
+        return relayRequest(request, url, headers) 
+    
+class AgreementProxy(Resource):
+    def post(self):
+        url = "https://apidgp-gameplayer.games.dmm.com/v5/agreement/confirm/client"
+        headers = {"User-Agent": "DMMGamePlayer5-Win/5.3.12 Electron/32.1.0",
+                "Client-App": "DMMGamePlayer5",
+                    "Client-version": "5.3.12",
+                    "Content-Type": "application/json"}
+
+        return relayRequest(request, url, headers) 

run.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+gunicorn --bind 127.0.0.1:8081 --config gunicorn_config.py wsgi:app

update.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+python -m modules.KF3.downloader