disabled stg server download

app.py

@@ -3,7 +3,6 @@ from flask_restful import Api
 
 from modules.KF3.database import Database as KF3DB
 from modules.Kingdom.database import Database as KFKDB
-from modules.Archive.database import Database as KFADB
 from modules.proxy import AgreementProxy, LaunchProxy, UpdateProxy
 
 app = Flask(__name__)
@@ -12,7 +11,6 @@ app.databases = {}
 
 api = Api(app)
 
-KFADB(api)
 api.add_resource(LaunchProxy, "/DMM/launch")
 api.add_resource(UpdateProxy, "/DMM/filelist", "/DMM/update")
 api.add_resource(AgreementProxy, "/DMM/agreement")

modules/Archive/database.py

@@ -1,142 +0,0 @@
-import json
-from flask import Flask
-
-
-from .databaseController import DatabaseController
-from .endpoints.query import Query
-from .endpoints.command import Command
-from .endpoints.commands import Commands
-from .endpoints.post import GetPost
-from .endpoints.posts import GetPosts
-from .endpoints.new_query import NewQuery
-from .endpoints.set_action import SetAction
-from .endpoints.posts_count import GetPostsCount
-from .endpoints.account_stats import AccountStats
-
-class Database:
-    db : DatabaseController = None
-    app : Flask = None
-
-    def __init__(self, api) -> None:
-        self.app = api.app
-
-        if "Archive" in self.app.databases:
-            del self.app.databases["Archive"]
-
-        self.reload_data("/home/pi/python/Katbots/JapariArchive/database.db")
-
-        self.app.databases["Archive"] = self
-
-        api.add_resource(Query, "/Archive/Query")
-        api.add_resource(NewQuery, "/Archive/NewQuery")
-        api.add_resource(Command, "/Archive/Command")
-        api.add_resource(Commands, "/Archive/Commands")
-        api.add_resource(GetPost, "/Archive/GetPost/<id>")
-        api.add_resource(GetPosts, "/Archive/GetPosts")
-        api.add_resource(GetPostsCount, "/Archive/GetPosts/Count")
-        api.add_resource(AccountStats, "/Archive/AccountStats")
-        api.add_resource(SetAction, "/Archive/SetAction")
-  
-    def get_accounts(self):
-        query = f'''
-        SELECT x_handle, id FROM accounts 
-        ORDER BY x_handle ASC'''
-
-        return self.db.run_query(query)
-
-    def get_account_stats(self):
-        query = '''SELECT x_handle,
-	SUM(IIF(x_posts.action_taken = 0, 1, 0)) as undecided,
-	SUM(IIF(x_posts.action_taken = 1, 1, 0)) as approved,
-	SUM(IIF(x_posts.action_taken = 2, 1, 0)) as deleted,
-	SUM(IIF(x_posts.action_taken = 3, 1, 0)) as hidden,
-	SUM(IIF(x_post_details.id is NULL, 1, 0)) as invalid
-FROM accounts 
-	LEFT JOIN x_posts on accounts.id = x_posts.account_id 
-	LEFT JOIN x_post_details on x_post_details.id = x_posts.id
-WHERE accounts.download_mode != 4
-GROUP BY account_id 
-ORDER BY x_handle'''
-        return self.db.run_query(query)
-
-    def get_post(self, id):
-        query = f'''SELECT x_posts.id, cast(x_posts.id as TEXT) as id_str, x_posts.error_id, x_posts.action_taken, x_posts.is_saved, x_post_details.text, x_post_details.files, x_post_details.date, accounts.x_handle, accounts.rating from x_posts
-        LEFT JOIN x_post_details
-            ON x_posts.id = x_post_details.id
-        LEFT JOIN accounts
-            ON x_posts.account_id = accounts.id WHERE x_posts.id = {id}
-        LIMIT 1'''
-        result = self.db.run_query(query)
-        if len(result) == 0:
-            return None
-        else:
-            #return most recent post
-            return result[-1]
-
-    def build_where_query(self, artist, actions_taken = [0, 1, 2, 3], last_id = -1, include_ratings = ["SFW", "NSFW"]):
-        where_query = "WHERE x_post_details.id NOT NULL AND x_posts.error_id = 0"
-        if last_id != -1:
-            where_query += f" AND x_posts.id < {last_id}"
-        if actions_taken != [0, 1, 2, 3]:
-            where_query += " AND (" + " OR ".join([f"x_posts.action_taken = {action}" for action in actions_taken]) + ")"
-        if include_ratings != ["SFW", "NSFW", "NSFL"]:
-            where_query += " AND (" + " OR ".join([f'accounts.rating = "{rating}"' for rating in include_ratings]) + ")"
-        if artist is not None and artist != "":
-            where_query += f' AND accounts.x_handle = "{artist}"'
-        return where_query
-
-    def get_posts_count(self, artist, actions_taken = [0, 1, 2, 3], last_id = -1, include_ratings = ["SFW", "NSFW"]):
-        where_query = self.build_where_query(artist, actions_taken, last_id, include_ratings)
-
-        query = f'''
-        SELECT count(*) as count FROM x_posts 
-        LEFT JOIN x_post_details
-            ON x_posts.id = x_post_details.id
-        LEFT JOIN accounts
-            ON x_posts.account_id = accounts.id
-        {where_query}'''
-
-        result = self.db.run_query(query)
-        return result[0]["count"]
-
-    def get_posts(self, num_posts, artist, actions_taken = [0, 1, 2, 3], last_id = -1, offset = 0, include_ratings = ["SFW", "NSFW"], order = "DESC"):
-        num_posts = max(1, min(num_posts, 100))
-
-        order_by = "RANDOM()" if order == "RAND" else "x_posts.id ASC" if order == "ASC" else "x_posts.id DESC"
-
-        where_query = self.build_where_query(artist, actions_taken, last_id, include_ratings)
-
-        query = f'''
-        SELECT x_posts.id, cast(x_posts.id as TEXT) as id_str, x_posts.action_taken, x_posts.is_saved, x_post_details.text, x_post_details.files, x_post_details.date, accounts.x_handle, accounts.rating FROM x_posts 
-        LEFT JOIN x_post_details
-            ON x_posts.id = x_post_details.id
-        LEFT JOIN accounts
-            ON x_posts.account_id = accounts.id
-        {where_query}
-        ORDER BY {order_by}
-        LIMIT {num_posts} OFFSET {offset}'''
-
-        return self.db.run_query(query)
-
-    def wrap_query_response(self, result, mode = "json"):
-        if result is None:
-            response = self.app.response_class(status=400)
-        else:
-            if mode == "json":
-                response = self.app.response_class(
-                    response=json.dumps(result, ensure_ascii=False, indent=1),
-                    status=200,
-                    mimetype='application/json'
-            )
-            elif mode == "text":
-                response = self.app.response_class(
-                    response=str(result),
-                    status=200,
-                    mimetype='text/plain'
-            )
-                
-        response.headers.add("Access-Control-Allow-Origin", "*")
-        return response
-
-    def reload_data(self, database_path):
-        self.db = DatabaseController(database_path)

modules/Archive/databaseController.py

@@ -1,48 +0,0 @@
-import os
-import sqlite3
-
-TABLE_ACCOUNTS  = "accounts"
-TABLE_X         = "x_posts"
-
-class DatabaseController:
-    def __init__(self, db_name):
-        self.conn = sqlite3.connect(db_name)
-        self.conn.row_factory = sqlite3.Row
-        self.cursor = self.conn.cursor()
-
-    def run_query(self, query):
-        try:
-            self.cursor.execute(query)
-            results = [dict(result) for result in self.cursor.fetchall()]
-            return results
-        except Exception as e:
-            print(e)
-            return []
-        
-    def run_command(self, command):
-        try:
-            self.cursor.execute(command)
-            result = self.cursor.rowcount
-            self.conn.commit()
-            return result > 0
-        except Exception as e:
-            self.conn.rollback()
-            print(command, e)
-            return False
-
-    def run_commands(self, commands):
-        last_command = ""
-        try:
-            for command in commands:
-                last_command = command
-                self.cursor.execute(command)
-            result = self.cursor.rowcount
-            self.conn.commit()
-            return result > 0
-        except Exception as e:
-            self.conn.rollback()
-            print(last_command, e)
-            return False
-        
-    def close(self):
-        self.conn.close()

modules/Archive/endpoints/account_stats.py

@@ -1,13 +0,0 @@
-from __future__ import annotations
-from flask_restful import Resource
-from flask import current_app as app
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from modules.Archive.database import Database
-
-class AccountStats(Resource):
-    def get(self):
-        db : Database = app.databases["Archive"]
-        result = db.get_account_stats()
-        return db.wrap_query_response(result)

modules/Archive/endpoints/command.py

@@ -1,23 +0,0 @@
-from __future__ import annotations
-from flask_restful import Resource
-from flask import current_app as app, request
-from typing import TYPE_CHECKING
-import hashlib
-
-if TYPE_CHECKING:
-    from modules.Archive.database import Database
-
-class Command(Resource):
-    def post(self):
-        db : Database = app.databases["Archive"]
-        auth = request.headers.get('auth')
-        if auth is not None:
-            hash_obj = hashlib.sha256(auth.encode('utf-8'))
-            if hash_obj.hexdigest() == "63a3b0dba950e1015a110486518e5ceff8cff415041aba3dedb8dc5aa3b3dd16":
-                query = request.data.decode("utf-8")
-                result = db.db.run_command(query)
-            else: 
-                result = None
-        else:
-            result = None
-        return db.wrap_query_response(result, mode="text")

modules/Archive/endpoints/commands.py

@@ -1,24 +0,0 @@
-from __future__ import annotations
-import json
-from flask_restful import Resource
-from flask import current_app as app, request
-from typing import TYPE_CHECKING
-import hashlib
-
-if TYPE_CHECKING:
-    from modules.Archive.database import Database
-
-class Commands(Resource):
-    def post(self):
-        db : Database = app.databases["Archive"]
-        auth = request.headers.get('auth')
-        if auth is not None:
-            hash_obj = hashlib.sha256(auth.encode('utf-8'))
-            if hash_obj.hexdigest() == "63a3b0dba950e1015a110486518e5ceff8cff415041aba3dedb8dc5aa3b3dd16":
-                data = json.loads(request.data)
-                result = db.db.run_commands(data)
-            else: 
-                result = None
-        else:
-            result = None
-        return db.wrap_query_response(result, mode="text")

modules/Archive/endpoints/new_query.py

@@ -1,17 +0,0 @@
-from __future__ import annotations
-from flask_restful import Resource
-from flask import current_app as app, request
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from modules.Archive.database import Database
-
-class NewQuery(Resource):
-    def post(self):
-        query = request.data.decode("utf-8")
-        
-        db : Database = app.databases["Archive"]
-
-        result = db.db.run_query(query)
-
-        return db.wrap_query_response(result)

modules/Archive/endpoints/post.py

@@ -1,15 +0,0 @@
-from __future__ import annotations
-from flask_restful import Resource
-from flask import current_app as app
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from modules.Archive.database import Database
-
-class GetPost(Resource):
-    def get(self, id):
-        id = int(id)
-        db : Database = app.databases["Archive"]
-
-        result = db.get_post(id)
-        return db.wrap_query_response(result) 

modules/Archive/endpoints/posts.py

@@ -1,55 +0,0 @@
-from __future__ import annotations
-from flask_restful import Resource
-from flask import current_app as app, request
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from modules.Archive.database import Database
-
-class GetPosts(Resource):
-    def get(self):
-        db : Database = app.databases["Archive"]
-        try:
-            count = int(request.args["count"]) if "count" in request.args else 20
-            page = int(request.args["page"]) if "page" in request.args else 1
-            artist = request.args["artist"] if "artist" in request.args else None
-            last_id = int(request.args["last_id"]) if "last_id" in request.args else -1
-        except:
-            response = app.response_class(status=400)
-            response.headers.add("Access-Control-Allow-Origin", "*")
-            return response   
-
-        actions = []
-        if "undecided" in request.args:
-            actions.append(0)
-        if "approved" in request.args:
-            actions.append(1)
-        if "denied" in request.args:
-            actions.append(2)
-        if "hidden" in request.args:
-            actions.append(3)
-        if actions == []:
-            actions = [0,1,2,3]
-
-        ratings = []
-        if "SFW" in request.args:
-            ratings.append("SFW")
-        if "NSFW" in request.args:
-            ratings.append("NSFW")
-        if "NSFL" in request.args:
-            ratings.append("NSFL")
-        if ratings == []:
-            ratings = ["SFW", "NSFW"]
-
-        if "random" in request.args:
-            order = "RAND"
-        elif "ascending" in request.args:
-            order = "ASC"
-        else:
-            order = "DESC"
-
-        real_page = page-1
-        offset = real_page * count
-
-        result = db.get_posts(count, artist, actions_taken=actions, last_id= -1, offset= offset, include_ratings= ratings, order=order)
-        return db.wrap_query_response(result)

modules/Archive/endpoints/posts_count.py

@@ -1,42 +0,0 @@
-from __future__ import annotations
-from flask_restful import Resource
-from flask import current_app as app, request
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from modules.Archive.database import Database
-
-class GetPostsCount(Resource):
-    def get(self):
-        db : Database = app.databases["Archive"]
-        try:
-            artist = request.args["artist"] if "artist" in request.args else None
-        except:
-            response = app.response_class(status=400)
-            response.headers.add("Access-Control-Allow-Origin", "*")
-            return response   
-
-        actions = []
-        if "undecided" in request.args:
-            actions.append(0)
-        if "approved" in request.args:
-            actions.append(1)
-        if "denied" in request.args:
-            actions.append(2)
-        if "hidden" in request.args:
-            actions.append(3)
-        if actions == []:
-            actions = [0,1,2,3]
-
-        ratings = []
-        if "SFW" in request.args:
-            ratings.append("SFW")
-        if "NSFW" in request.args:
-            ratings.append("NSFW")
-        if "NSFL" in request.args:
-            ratings.append("NSFL")
-        if ratings == []:
-            ratings = ["SFW", "NSFW"]
-
-        result = db.get_posts_count(artist, actions_taken=actions, last_id= -1, include_ratings= ratings)
-        return db.wrap_query_response(result, mode="text") 

modules/Archive/endpoints/query.py

@@ -1,18 +0,0 @@
-from __future__ import annotations
-from flask_restful import Resource
-from flask import current_app as app, request
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from modules.Archive.database import Database
-
-class Query(Resource):
-    def post(self):
-        query = request.data.decode("utf-8")
-        
-        db : Database = app.databases["Archive"]
-
-        result = db.db.run_query(query)
-        result = [list(d.values()) for d in result]
-
-        return db.wrap_query_response(result)

modules/Archive/endpoints/set_action.py

@@ -1,35 +0,0 @@
-from __future__ import annotations
-import json
-from flask_restful import Resource
-from flask import current_app as app, request
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from modules.Archive.database import Database
-
-class SetAction(Resource):
-    def post(self):
-        data = json.loads(request.data)
-
-        try:
-            bypass = ("allow_override" in data) and data["allow_override"]
-            action = data["action_taken"]
-            if "id_str" in data:
-                id = int(data["id_str"])
-            elif "id" in data:
-                id = data["id"]
-            else:
-                raise Exception("no id (int) or id_str (str) specified")
-        except Exception as e:
-            print(e)
-            response = app.response_class(response=e, status=400)
-            response.headers.add("Access-Control-Allow-Origin", "*")
-            return response
-
-        db : Database = app.databases["Archive"]
-        query = f"UPDATE x_posts SET action_taken = {action} WHERE id = {id}"
-        if not bypass:
-            query += " AND action_taken = 0"
-
-        result = db.db.run_command(query)
-        return db.wrap_query_response(result, mode="text")

modules/Archive/load_from_file.py

@@ -1,8 +0,0 @@
-import sys
-from databaseController import DatabaseController
-
-db = DatabaseController("/home/pi/python/Katbots/JapariArchive/database.db")
-with open(sys.argv[1], "rt") as file:
-    lines = [line for line in file.readlines() if line.strip()]
-
-print(db.run_commands(lines))

modules/KF3/downloader.py

@@ -239,24 +239,24 @@ async def manual():
     downloaded_cache = {}
     downloaded_files = {}
     
-    async with aiohttp.ClientSession() as session:
-        param = encode(json.dumps({"version":"1.0.0","dmm_viewer_id":0,"platform":1}))
-        request = await download_bytes(servers[0] + "paradesv/common/GetUrl.do?param=" + param, session)
-        result  = gzip.decompress(request)
-        response = json.loads(result)
-        asset_bundle_url = response["asset_bundle_url"]
-        urlName = asset_bundle_url.split("-")[2]
+    # async with aiohttp.ClientSession() as session:
+    #     param = encode(json.dumps({"version":"1.0.0","dmm_viewer_id":0,"platform":1}))
+    #     request = await download_bytes(servers[0] + "paradesv/common/GetUrl.do?param=" + param, session)
+    #     result  = gzip.decompress(request)
+    #     response = json.loads(result)
+    #     asset_bundle_url = response["asset_bundle_url"]
+    #     urlName = asset_bundle_url.split("-")[2]
 
-    print("downloading from", servers[0])
-    downloaded_cache = await download_cache(urlName, servers[0])
-    downloaded_files = await download_files(urlName, asset_bundle_url, "Windows")
+    # print("downloading from", servers[0])
+    # downloaded_cache = await download_cache(urlName, servers[0])
+    # downloaded_files = await download_files(urlName, asset_bundle_url, "Windows")
 
-    if downloaded_cache != [] and downloaded_cache != None:
-        with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_cache.json", "wt", encoding="utf-8") as file:
-            json.dump(downloaded_cache, file, ensure_ascii=False, indent=1)
-    if downloaded_files != [] and downloaded_files != None:
-        with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_files.json", "wt", encoding="utf-8") as file:
-            json.dump(downloaded_files, file, ensure_ascii=False, indent=1)
+    # if downloaded_cache != [] and downloaded_cache != None:
+    #     with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_cache.json", "wt", encoding="utf-8") as file:
+    #         json.dump(downloaded_cache, file, ensure_ascii=False, indent=1)
+    # if downloaded_files != [] and downloaded_files != None:
+    #     with open("/var/www/html/Katworks/KF/assets/KF3/lastUpdate_prod_files.json", "wt", encoding="utf-8") as file:
+    #         json.dump(downloaded_files, file, ensure_ascii=False, indent=1)
 
     print("downloading from", servers[1])
     asset_bundle_url = "https://parade-mobile-develop01-app.kemono-friends-3.jp/AssetBundles/0.0.0/latest"