fresh start

2026-01-08 22:07:03 +01:00
commit 715be97289
30 changed files with 2302 additions and 0 deletions
--- a/Twitter/tweetyapi.py
+++ b/Twitter/tweetyapi.py
@@ -0,0 +1,102 @@
+import asyncio
+from typing import AsyncIterator
+from tweety import TwitterAsync
+from tweety.types.twDataTypes import Tweet, SelfThread, ConversationThread
+from tweety.types.usertweet import UserMedia
+from tweety.exceptions_ import RateLimitReached
+from config import Global_Config
+
+class TweetyApi:
+    async def init(self, skip_login = False, session_name = "session"):
+        if skip_login:
+            self.app = TwitterAsync(session_name)
+        else:
+            if Global_Config["x_cookies"] == None:
+                raise Exception("X cookies are required")
+            cookies = Global_Config["x_cookies"]
+            self.app = TwitterAsync(session_name)
+            await self.app.load_cookies(cookies)
+        print(self.app.user)
+        return self
+
+    async def get_tweet(self, url):
+        try:
+            tweet = await self.app.tweet_detail(url)
+            return tweet
+        except:
+            return None
+
+    async def get_tweets(self, user_name, bottom_id, all_posts : list) -> AsyncIterator[Tweet]:
+        def validate_tweet(tweet : Tweet):
+            tweet_id_num = int(tweet.id)
+
+            past_bounds = False
+            tweet_valid = True
+
+            if tweet_id_num <= bottom_id:
+                past_bounds = True
+            if tweet_id_num in all_posts:
+                tweet_valid = False
+
+            return past_bounds, tweet_valid
+            
+        sleep_default = 0.125
+        sleep_exponent = 1
+        user = None
+
+        while user == None:
+            try:
+                user = await self.app.get_user_info(username=user_name)
+            except RateLimitReached as ex:
+                sleep_exponent = await self.sleep_wait(sleep_default, sleep_exponent)
+            except Exception as ex:
+                print("User error: " + str(ex))
+                raise ex
+
+        tweety_api = UserMedia(user.rest_id, self.app, 1, 2, None)
+        sleep_exponent = 1
+
+        while True:
+            await asyncio.sleep(5)
+            old_cursor = tweety_api.cursor
+            
+            try:
+                tweets = await tweety_api.get_next_page()
+                sleep_exponent = 1
+            except RateLimitReached as ex:
+                sleep_exponent = await self.sleep_wait(sleep_default, sleep_exponent)
+                tweety_api.cursor = old_cursor
+                continue
+            except Exception as ex:
+                raise ex
+
+            has_valid_tweets = False
+            for tweet in tweets:
+                if isinstance(tweet, ConversationThread) | isinstance(tweet, SelfThread):
+                    tweet:ConversationThread | SelfThread
+                    for tweet1 in tweet.tweets:
+                        _, tweet_valid = validate_tweet(tweet1)
+                        if tweet_valid: 
+                            has_valid_tweets = True 
+                            yield tweet1
+                else:
+                    past_bounds, tweet_valid = validate_tweet(tweet)
+                    if past_bounds: continue
+                    if tweet_valid:
+                        has_valid_tweets = True 
+                        yield tweet
+                        
+            if len(tweets) == 0 or not has_valid_tweets:
+                break
+            await asyncio.sleep(1)
+
+    @staticmethod
+    async def sleep_wait(sleep_default, sleep_exponent):
+        sleep_amount = min(sleep_default * pow(2,sleep_exponent), 2)
+        print(f"Sleeping for {round(sleep_amount,2)} hours.")
+        await asyncio.sleep(sleep_amount * 60 * 60)
+        print("Sleep done")
+        sleep_exponent += 1
+        return sleep_exponent
+
+#asyncio.run(TweetyApi().get_tweets("redhood_depth", 0))