You've already forked JapariArchive
fresh start
This commit is contained in:
37
Pixiv/downloader.py
Normal file
37
Pixiv/downloader.py
Normal file
@@ -0,0 +1,37 @@
|
||||
from __future__ import annotations
|
||||
import traceback
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import nextcord
|
||||
|
||||
from Discord import discordHelper
|
||||
from config import Global_Config
|
||||
if TYPE_CHECKING:
|
||||
from runtimeBotData import RuntimeBotData
|
||||
|
||||
async def download_loop(botData: RuntimeBotData):
|
||||
try:
|
||||
db = botData.db
|
||||
guild = botData.client.guilds[0]
|
||||
|
||||
new_posts = await botData.pixivApi.get_new_posts(botData)
|
||||
new_posts.sort(key= lambda x: x.id)
|
||||
|
||||
for post in new_posts:
|
||||
media = (await botData.pixivApi.download_illust(post, Global_Config["pixiv_download_path"]))[:4]
|
||||
if post.x_restrict == 0:
|
||||
channel = nextcord.utils.get(guild.channels, name="pixiv-sfw-feed")
|
||||
elif post.x_restrict == 1:
|
||||
channel = nextcord.utils.get(guild.channels, name="pixiv-r18-feed")
|
||||
else:
|
||||
channel = nextcord.utils.get(guild.channels, name="pixiv-r18g-feed")
|
||||
await discordHelper.send_pixiv_post(post, media, channel)
|
||||
|
||||
if db.pixiv_get_user(post.user.id) == None:
|
||||
db.pixiv_insert_user(post.user)
|
||||
db.pixiv_insert_post(post)
|
||||
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
await discordHelper.send_error(traceback.format_exc()[0:256], botData)
|
||||
print("Pixiv done")
|
||||
230
Pixiv/pixivapi.py
Normal file
230
Pixiv/pixivapi.py
Normal file
@@ -0,0 +1,230 @@
|
||||
from __future__ import annotations
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from config import Global_Config
|
||||
if TYPE_CHECKING:
|
||||
from runtimeBotData import RuntimeBotData
|
||||
|
||||
import os
|
||||
import asyncio
|
||||
from io import BytesIO
|
||||
from zipfile import ZipFile
|
||||
from pixivpy3 import AppPixivAPI
|
||||
from Twitter.tweetyapi import TweetyApi
|
||||
from getPixivpyToken.gppt import GetPixivToken
|
||||
from PIL import Image
|
||||
|
||||
class PixivApi:
|
||||
def init(self):
|
||||
self.api = AppPixivAPI(timeout=60)
|
||||
self.api.auth(refresh_token=self.get_refresh_token())
|
||||
return self
|
||||
|
||||
def get_refresh_token(self) -> str:
|
||||
if Global_Config["pixiv_token"] is not None:
|
||||
return Global_Config["pixiv_token"]
|
||||
else:
|
||||
if Global_Config["pixiv_username"] is None or Global_Config["pixiv_password"] is None:
|
||||
raise Exception("Pixiv username and password are required")
|
||||
g = GetPixivToken(headless=True, username=Global_Config["pixiv_username"], password=Global_Config["pixiv_password"])
|
||||
refresh_token = g.login()["refresh_token"]
|
||||
Global_Config["pixiv_token"] = refresh_token
|
||||
return refresh_token
|
||||
|
||||
async def search_illust(self, retries = 3, **next_qs):
|
||||
last_error = ""
|
||||
sleep_default = 0.125
|
||||
for i in range(retries):
|
||||
try:
|
||||
json_result = self.api.search_illust(**next_qs)
|
||||
|
||||
if json_result.error != None:
|
||||
raise Exception(json_result.error)
|
||||
|
||||
return json_result
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
last_error = ex
|
||||
await TweetyApi.sleep_wait(sleep_default, i)
|
||||
self.api = PixivApi().init().api
|
||||
continue
|
||||
|
||||
raise Exception(last_error)
|
||||
|
||||
async def illust_detail(self, id, retries = 3):
|
||||
last_error = ""
|
||||
sleep_default = 0.125
|
||||
for i in range(retries):
|
||||
try:
|
||||
json_result = self.api.illust_detail(id)
|
||||
|
||||
if json_result.error != None:
|
||||
if json_result.error.user_message == 'Page not found':
|
||||
return None
|
||||
if json_result.error.user_message == 'Artist has made their work private.':
|
||||
return None
|
||||
raise Exception(json_result.error)
|
||||
|
||||
return json_result
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
last_error = ex
|
||||
await TweetyApi.sleep_wait(sleep_default, i)
|
||||
self.api = PixivApi().init().api
|
||||
continue
|
||||
|
||||
raise Exception(last_error)
|
||||
|
||||
async def ugoira_metadata(self, id, retries = 3):
|
||||
last_error = ""
|
||||
sleep_default = 0.125
|
||||
for i in range(retries):
|
||||
try:
|
||||
json_result = self.api.ugoira_metadata(id)
|
||||
|
||||
if json_result.error != None:
|
||||
if json_result.error.user_message == 'Page not found':
|
||||
return None
|
||||
if json_result.error.user_message == 'Artist has made their work private.':
|
||||
return None
|
||||
raise Exception(json_result.error)
|
||||
|
||||
return json_result
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
last_error = ex
|
||||
await TweetyApi.sleep_wait(sleep_default, i)
|
||||
self.api = PixivApi().init().api
|
||||
continue
|
||||
|
||||
raise Exception(last_error)
|
||||
|
||||
async def download(self, url, retries = 3):
|
||||
for i in range(retries):
|
||||
try:
|
||||
with BytesIO() as io_bytes:
|
||||
def foo():
|
||||
self.api.download(url, fname=io_bytes)
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(None, foo)
|
||||
return io_bytes.getbuffer().tobytes()
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
if i == 3:
|
||||
raise ex
|
||||
await asyncio.sleep(i * 60)
|
||||
|
||||
async def get_new_posts(self, botData: RuntimeBotData):
|
||||
posts = []
|
||||
# get all page:
|
||||
next_qs = {'word': 'けものフレンズ', 'search_target': 'partial_match_for_tags', 'sort': 'date_desc', 'filter': 'for_ios', 'end_date': ''}
|
||||
|
||||
page = 0
|
||||
while next_qs:
|
||||
print("Getting pixiv page")
|
||||
has_new_posts = False
|
||||
|
||||
json_result = await self.search_illust(retries=3, **next_qs)
|
||||
|
||||
for illust in json_result.illusts:
|
||||
if botData.db.pixiv_get_post(illust.id) == None:
|
||||
has_new_posts = True
|
||||
print(illust.id, illust.title, illust.create_date)
|
||||
posts.append(illust)
|
||||
|
||||
if has_new_posts == False:
|
||||
break
|
||||
|
||||
next_qs = self.api.parse_qs(json_result.next_url)
|
||||
|
||||
if len(json_result.illusts) > 0:
|
||||
oldest_date = json_result.illusts[-1].create_date[0:10]
|
||||
if "end_date" in next_qs and oldest_date != next_qs["end_date"]:
|
||||
next_qs["end_date"] = oldest_date
|
||||
next_qs["offset"] = sum(1 for x in json_result.illusts if x.create_date[0:10] == oldest_date)
|
||||
|
||||
page += 1
|
||||
await asyncio.sleep(1)
|
||||
|
||||
return posts
|
||||
|
||||
async def download_illust(self, illust, download_path = "Temp/"):
|
||||
print(illust.id)
|
||||
downloaded = []
|
||||
filepath = os.path.abspath(os.path.join(download_path, str(illust.user.id)))
|
||||
os.makedirs(filepath, exist_ok=True)
|
||||
|
||||
if illust.type == 'ugoira':
|
||||
filename = str(illust.id) + "_p0.gif"
|
||||
filepath = os.path.join(filepath, filename)
|
||||
bytes = await self.download_ugoira_to_bytes(illust.id)
|
||||
if bytes is None:
|
||||
return []
|
||||
with open(filepath, "wb") as f:
|
||||
f.write(bytes)
|
||||
downloaded.append({"file":bytes, "name":filename})
|
||||
else:
|
||||
if len(illust.meta_pages) == 0:
|
||||
_, filename = os.path.split(illust.meta_single_page.original_image_url)
|
||||
filepath = os.path.join(filepath, filename)
|
||||
bytes = await self.download(illust.meta_single_page.original_image_url)
|
||||
with open(filepath, "wb") as f:
|
||||
f.write(bytes)
|
||||
if len(downloaded) < 4:
|
||||
downloaded.append({"file":bytes, "name":filename})
|
||||
else:
|
||||
for page in illust.meta_pages:
|
||||
filepath = os.path.abspath(os.path.join(download_path, str(illust.user.id)))
|
||||
_, filename = os.path.split(page.image_urls.original)
|
||||
filepath = os.path.join(filepath, filename)
|
||||
bytes = await self.download(page.image_urls.original)
|
||||
with open(filepath, "wb") as f:
|
||||
f.write(bytes)
|
||||
if len(downloaded) < 4:
|
||||
downloaded.append({"file":bytes, "name":filename})
|
||||
|
||||
return downloaded
|
||||
|
||||
async def download_ugoira_to_bytes(self, id):
|
||||
closables = []
|
||||
|
||||
metadata = await self.ugoira_metadata(id)
|
||||
if metadata is None:
|
||||
return None
|
||||
|
||||
if len(metadata.ugoira_metadata.zip_urls) > 1:
|
||||
raise Exception
|
||||
|
||||
bytes = await self.download(metadata.ugoira_metadata.zip_urls["medium"])
|
||||
with BytesIO(bytes) as bytes0, ZipFile(bytes0) as input_zip:
|
||||
frames = metadata.ugoira_metadata.frames
|
||||
zip_frames = {}
|
||||
for name in input_zip.namelist():
|
||||
im = input_zip.read(name)
|
||||
im_bytes = BytesIO(im)
|
||||
im_im = Image.open(im_bytes)
|
||||
closables += [im_bytes, im_im]
|
||||
zip_frames[name] = im_im
|
||||
|
||||
with BytesIO() as buffer:
|
||||
zip_frames[frames[0].file].save(
|
||||
buffer,
|
||||
format="GIF",
|
||||
save_all=True,
|
||||
append_images=[zip_frames[frame.file] for frame in frames[1:]],
|
||||
duration=[frame.delay for frame in frames],
|
||||
loop=0,
|
||||
optimize=False
|
||||
)
|
||||
|
||||
[closable.close() for closable in closables]
|
||||
return buffer.getbuffer().tobytes()
|
||||
|
||||
if __name__ == "__main__":
|
||||
async def get_new_posts():
|
||||
api = PixivApi().init()
|
||||
while True:
|
||||
await api.get_new_posts()
|
||||
await asyncio.sleep(30 * 60)
|
||||
|
||||
asyncio.run(get_new_posts())
|
||||
Reference in New Issue
Block a user