katboi01/KemoFureApi
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

removed katworks api files

Browse Source 
added postgresql database
This commit is contained in:
katboi01
2025-07-17 02:37:27 +02:00
parent 11edbf3abf
commit f2909d2992
32 changed files with 99 additions and 1900 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
modules/Archive/database.py
View File

@@ -1,6 +1,7 @@
import json
from flask import Flask
from modules.Archive.endpoints.tag_stats import TagStats
from .databaseController import DatabaseController
from .endpoints.query import Query
@@ -23,108 +24,93 @@ class Database:
if "Archive" in self.app.databases:
del self.app.databases["Archive"]
self.reload_data("/home/pi/python/Katbots/JapariArchive/database.db")
self.reload_data()
self.app.databases["Archive"] = self
api.add_resource(Query, "/Archive/Query")
api.add_resource(NewQuery, "/Archive/NewQuery")
api.add_resource(Command, "/Archive/Command")
api.add_resource(Commands, "/Archive/Commands")
api.add_resource(GetPost, "/Archive/GetPost/<id>")
api.add_resource(GetPosts, "/Archive/GetPosts")
api.add_resource(GetPostsCount, "/Archive/GetPosts/Count")
api.add_resource(AccountStats, "/Archive/AccountStats")
api.add_resource(SetAction, "/Archive/SetAction")
api.add_resource(Query, "/Query")
api.add_resource(NewQuery, "/NewQuery")
api.add_resource(Command, "/Command")
api.add_resource(Commands, "/Commands")
api.add_resource(GetPost, "/GetPost/<id>")
api.add_resource(GetPosts, "/GetPosts")
api.add_resource(GetPostsCount, "/GetPosts/Count")
api.add_resource(AccountStats, "/AccountStats")
api.add_resource(TagStats, "/TagStats")
api.add_resource(SetAction, "/SetAction")
def get_accounts(self):
query = f'''
SELECT x_handle, id FROM accounts
SELECT x_handle, id FROM x_accounts
ORDER BY x_handle ASC'''
return self.db.run_query(query)
def get_account_stats(self):
query = '''SELECT x_handle,
SUM(IIF(x_posts.action_taken = 0, 1, 0)) as undecided,
SUM(IIF(x_posts.action_taken = 1, 1, 0)) as approved,
SUM(IIF(x_posts.action_taken = 2, 1, 0)) as deleted,
SUM(IIF(x_posts.action_taken = 3, 1, 0)) as hidden,
SUM(IIF(x_post_details.id is NULL, 1, 0)) as invalid
FROM accounts
LEFT JOIN x_posts on accounts.id = x_posts.account_id
LEFT JOIN x_post_details on x_post_details.id = x_posts.id
WHERE accounts.download_mode != 4
GROUP BY account_id
ORDER BY x_handle'''
query = 'SELECT * FROM "Artist Stats"'
return self.db.run_query(query)
def get_tag_stats(self):
query = 'SELECT * FROM "Tag Stats"'
return self.db.run_query(query)
def get_post(self, id):
query = f'''SELECT x_posts.id, cast(x_posts.id as TEXT) as id_str, x_posts.error_id, x_posts.action_taken, x_posts.is_saved, x_post_details.text, x_post_details.files, x_post_details.date, accounts.x_handle, accounts.rating from x_posts
LEFT JOIN x_post_details
ON x_posts.id = x_post_details.id
LEFT JOIN accounts
ON x_posts.account_id = accounts.id WHERE x_posts.id = {id}
query = f'''SELECT cast(x_posts.id as TEXT), x_posts.*, x_accounts.x_handle, x_accounts.rating from x_posts
LEFT JOIN x_accounts
ON x_posts.account_id = x_accounts.id WHERE x_posts.id = {id}
LIMIT 1'''
result = self.db.run_query(query)
if len(result) == 0:
return None
else:
#return most recent post
return result[-1]
return self.db.run_query(query)
def build_where_query(self, artist, actions_taken = [0, 1, 2, 3], last_id = -1, include_ratings = ["SFW", "NSFW"]):
where_query = "WHERE x_post_details.id NOT NULL AND x_posts.error_id = 0"
def build_where_query(self, artist, actions_taken = [0, 1, 2, 3], last_id = -1, include_ratings = ["SFW", "NSFW"], tags = []):
where_query = "WHERE x_posts.error_id = 0"
if last_id != -1:
where_query += f" AND x_posts.id < {last_id}"
if actions_taken != [0, 1, 2, 3]:
where_query += " AND (" + " OR ".join([f"x_posts.action_taken = {action}" for action in actions_taken]) + ")"
if include_ratings != ["SFW", "NSFW", "NSFL"]:
where_query += " AND (" + " OR ".join([f'accounts.rating = "{rating}"' for rating in include_ratings]) + ")"
where_query += " AND (" + " OR ".join([f'x_accounts.rating = \'{rating}\'' for rating in include_ratings]) + ")"
if artist is not None and artist != "":
where_query += f' AND accounts.x_handle = "{artist}"'
where_query += f' AND x_accounts.x_handle = \'{artist}\''
if len(tags) > 0:
tags = ", ".join(["'" + tag + "'" for tag in tags])
where_query += f' AND tags @> ARRAY[{tags}]'
return where_query
def get_posts_count(self, artist, actions_taken = [0, 1, 2, 3], last_id = -1, include_ratings = ["SFW", "NSFW"]):
where_query = self.build_where_query(artist, actions_taken, last_id, include_ratings)
def get_posts_count(self, artist, actions_taken = [0, 1, 2, 3], last_id = -1, include_ratings = ["SFW", "NSFW"], tags = []):
where_query = self.build_where_query(artist, actions_taken, last_id, include_ratings, tags)
query = f'''
SELECT count(*) as count FROM x_posts
LEFT JOIN x_post_details
ON x_posts.id = x_post_details.id
LEFT JOIN accounts
ON x_posts.account_id = accounts.id
LEFT JOIN x_accounts
ON x_posts.account_id = x_accounts.id
{where_query}'''
result = self.db.run_query(query)
_, result = self.db.run_query(query)
return result[0]["count"]
def get_posts(self, num_posts, artist, actions_taken = [0, 1, 2, 3], last_id = -1, offset = 0, include_ratings = ["SFW", "NSFW"], order = "DESC"):
def get_posts(self, num_posts, artist, actions_taken = [0, 1, 2, 3], last_id = -1, offset = 0, include_ratings = ["SFW", "NSFW"], tags = [], order = "DESC"):
num_posts = max(1, min(num_posts, 100))
order_by = "RANDOM()" if order == "RAND" else "x_posts.id ASC" if order == "ASC" else "x_posts.id DESC"
where_query = self.build_where_query(artist, actions_taken, last_id, include_ratings)
where_query = self.build_where_query(artist, actions_taken, last_id, include_ratings, tags)
query = f'''
SELECT x_posts.id, cast(x_posts.id as TEXT) as id_str, x_posts.action_taken, x_posts.is_saved, x_post_details.text, x_post_details.files, x_post_details.date, accounts.x_handle, accounts.rating FROM x_posts
LEFT JOIN x_post_details
ON x_posts.id = x_post_details.id
LEFT JOIN accounts
ON x_posts.account_id = accounts.id
SELECT x_posts.id, cast(x_posts.id as TEXT) as id_str, action_taken, saved_files != \'{{}}\' as is_saved, text, files, date, x_handle, x_accounts.rating FROM x_posts
LEFT JOIN x_accounts
ON x_posts.account_id = x_accounts.id
{where_query}
ORDER BY {order_by}
LIMIT {num_posts} OFFSET {offset}'''
return self.db.run_query(query)
def wrap_query_response(self, result, mode = "json"):
def wrap_query_response(self, result, mode = "json", error : str = ""):
if result is None:
response = self.app.response_class(status=400)
response = self.app.response_class(response=error, status=400)
else:
if mode == "json":
response = self.app.response_class(
response=json.dumps(result, ensure_ascii=False, indent=1),
response=json.dumps(result, ensure_ascii=False, indent=1, default=str),
status=200,
mimetype='application/json'
)
@@ -138,5 +124,5 @@ ORDER BY x_handle'''
response.headers.add("Access-Control-Allow-Origin", "*")
return response
def reload_data(self, database_path):
self.db = DatabaseController(database_path)
def reload_data(self):
self.db = DatabaseController()

24
modules/Archive/databaseController.py
View File

@@ -1,23 +1,29 @@
import os
import sqlite3
import psycopg
from psycopg.rows import dict_row
TABLE_ACCOUNTS = "accounts"
TABLE_ACCOUNTS = "x_accounts"
TABLE_X = "x_posts"
class DatabaseController:
def __init__(self, db_name):
self.conn = sqlite3.connect(db_name)
self.conn.row_factory = sqlite3.Row
def __init__(self):
if not os.path.exists("database_config.txt"):
raise Exception(f"Create 'database_config.txt' in {os.path.abspath(os.curdir)} with database connection string, example: 'dbname=japariarchive user=postgres password=password123'")
with open("database_config.txt", "rt") as file:
connection_string = file.read()
self.conn = psycopg.connect(connection_string, row_factory=dict_row)
self.cursor = self.conn.cursor()
def run_query(self, query):
try:
self.cursor.execute(query)
results = [dict(result) for result in self.cursor.fetchall()]
return results
results = self.cursor.fetchall()
self.conn.rollback()
return True, results
except Exception as e:
print(e)
return []
print(query, e)
self.conn.rollback()
return False, str(e)
def run_command(self, command):
try:

2
modules/Archive/endpoints/account_stats.py
View File

@@ -9,5 +9,5 @@ if TYPE_CHECKING:
class AccountStats(Resource):
def get(self):
db : Database = app.databases["Archive"]
result = db.get_account_stats()
_, result = db.get_account_stats()
return db.wrap_query_response(result)

8
modules/Archive/endpoints/new_query.py
View File

@@ -12,6 +12,8 @@ class NewQuery(Resource):
db : Database = app.databases["Archive"]
result = db.db.run_query(query)
return db.wrap_query_response(result)
status, result = db.db.run_query(query)
if status:
return db.wrap_query_response(result)
else:
return db.wrap_query_response(None, error = result)

2
modules/Archive/endpoints/post.py
View File

@@ -11,5 +11,5 @@ class GetPost(Resource):
id = int(id)
db : Database = app.databases["Archive"]
result = db.get_post(id)
_, result = db.get_post(id)
return db.wrap_query_response(result)

8
modules/Archive/endpoints/posts.py
View File

@@ -14,6 +14,7 @@ class GetPosts(Resource):
page = int(request.args["page"]) if "page" in request.args else 1
artist = request.args["artist"] if "artist" in request.args else None
last_id = int(request.args["last_id"]) if "last_id" in request.args else -1
tags = request.args["tags"].split() if "tags" in request.args else []
except:
response = app.response_class(status=400)
response.headers.add("Access-Control-Allow-Origin", "*")
@@ -51,5 +52,8 @@ class GetPosts(Resource):
real_page = page-1
offset = real_page * count
result = db.get_posts(count, artist, actions_taken=actions, last_id= -1, offset= offset, include_ratings= ratings, order=order)
return db.wrap_query_response(result)
status, result = db.get_posts(count, artist, actions_taken=actions, last_id= -1, offset= offset, include_ratings= ratings, order=order, tags=tags)
if status:
return db.wrap_query_response(result)
else:
return db.wrap_query_response(None, error=result)

3
modules/Archive/endpoints/posts_count.py
View File

@@ -11,6 +11,7 @@ class GetPostsCount(Resource):
db : Database = app.databases["Archive"]
try:
artist = request.args["artist"] if "artist" in request.args else None
tags = request.args["tags"].split() if "tags" in request.args else []
except:
response = app.response_class(status=400)
response.headers.add("Access-Control-Allow-Origin", "*")
@@ -38,5 +39,5 @@ class GetPostsCount(Resource):
if ratings == []:
ratings = ["SFW", "NSFW"]
result = db.get_posts_count(artist, actions_taken=actions, last_id= -1, include_ratings= ratings)
result = db.get_posts_count(artist, actions_taken=actions, last_id= -1, include_ratings= ratings, tags=tags)
return db.wrap_query_response(result, mode="text")

10
modules/Archive/endpoints/query.py
View File

@@ -12,7 +12,9 @@ class Query(Resource):
db : Database = app.databases["Archive"]
result = db.db.run_query(query)
result = [list(d.values()) for d in result]
return db.wrap_query_response(result)
status, result = db.db.run_query(query)
if status:
result = [list(d.values()) for d in result]
return db.wrap_query_response(result)
else:
return db.wrap_query_response(None, error = result)

13
modules/Archive/endpoints/tag_stats.py Normal file
View File

@@ -0,0 +1,13 @@
from __future__ import annotations
from flask_restful import Resource
from flask import current_app as app
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from modules.Archive.database import Database
class TagStats(Resource):
def get(self):
db : Database = app.databases["Archive"]
_, result = db.get_tag_stats()
return db.wrap_query_response(result)