From cb4f01b50499c1a589905cd99bccda5dd48cea65 Mon Sep 17 00:00:00 2001 From: Toru Furukawa Date: Mon, 5 Mar 2018 12:56:21 +0900 Subject: [PATCH 001/139] Add endpoint to delete webhook URL --- TwitterAPI/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 69b755f..ba1be8f 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -205,6 +205,7 @@ # NEW ACCOUNT ACTIVITY WEBHOOK API (BETA) 'account_activity/all/:PARAM/webhooks': ('POST', 'api'), + 'account_activity/all/:PARAM/webhooks/:PARAM': ('DELETE', 'api'), 'account_activity/all/:PARAM/subscriptions': ('POST', 'api'), 'account_activity/all/:PARAM/subscriptions/all': ('GET', 'api'), 'account_activity/all/:PARAM/subscriptions/all/list': ('GET', 'api'), From 1767b04092be4a39817ed298017e5feddec4709a Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 5 Mar 2018 13:58:28 -0500 Subject: [PATCH 002/139] Merged torufurukawa patch. Added webhook endpoint --- CHANGES.txt | 2 ++ TwitterAPI/__init__.py | 2 +- TwitterAPI/constants.py | 18 +++++++++--------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 48de199..cb4190c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -106,3 +106,5 @@ v2.4.6, 16 Jul 2017 -- Merged RickRedSix patch. Added direct message endpoint v2.4.7, 22 Oct 2017 -- Added endpoints for Ads API. v2.4.8, 03 Feb 2018 -- Merged RickRedSix patch. Added account activity endpoints. + +v2.4.9, 03 Feb 2018 -- Merged torufurukawa patch. Added webhook endpoint. diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index cf4921b..d501c56 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.4.8' +__version__ = '2.4.9' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index ba1be8f..5724995 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -204,16 +204,16 @@ # NEW ACCOUNT ACTIVITY WEBHOOK API (BETA) - 'account_activity/all/:PARAM/webhooks': ('POST', 'api'), - 'account_activity/all/:PARAM/webhooks/:PARAM': ('DELETE', 'api'), - 'account_activity/all/:PARAM/subscriptions': ('POST', 'api'), - 'account_activity/all/:PARAM/subscriptions/all': ('GET', 'api'), - 'account_activity/all/:PARAM/subscriptions/all/list': ('GET', 'api'), - 'account_activity/all/:PARAM/subscriptions': ('DELETE', 'api'), + 'account_activity/all/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions/all': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions/all/list': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions': ('DELETE', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/webhooks': ('POST', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/webhooks/:PARAM': ('DELETE', 'api'), # ENVIRONMENT NAME, WEBHOOK ID 'account_activity/all/count': ('GET', 'api'), 'account_activity/all/webhooks': ('GET', 'api'), 'account_activity/webhooks': ('POST', 'api'), - 'account_activity/webhooks/:PARAM': ('DELETE', 'api'), - 'account_activity/webhooks/:PARAM/subscriptions': ('POST', 'api'), - 'account_activity/webhooks/:PARAM/subscriptions/list': ('GET', 'api') + 'account_activity/webhooks/:PARAM': ('DELETE', 'api'), # WEBHOOK ID + 'account_activity/webhooks/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME + 'account_activity/webhooks/:PARAM/subscriptions/list': ('GET', 'api') # ENVIRONMENT NAME } From 1672da71157806e3e49a91ad024a9edbadd4cae7 Mon Sep 17 00:00:00 2001 From: Toru Furukawa Date: Mon, 12 Mar 2018 11:17:57 +0900 Subject: [PATCH 003/139] Add direct message API endopoint --- TwitterAPI/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index ba1be8f..51d96cc 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -67,6 +67,7 @@ 'direct_messages/sent': ('GET', 'api'), 'direct_messages/show': ('GET', 'api'), 'direct_messages/welcome_messages/new': ('POST', 'api'), + 'direct_messages/welcome_messages/list': ('GET', 'api'), 'direct_messages/welcome_messages/show': ('GET', 'api'), 'direct_messages/welcome_messages/destroy': ('DELETE', 'api'), 'direct_messages/welcome_messages/rules/new': ('POST', 'api'), From ec3043756c9109143381d2af57012337f382815a Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 12 Mar 2018 07:19:31 -0400 Subject: [PATCH 004/139] Updated endoints --- CHANGES.txt | 2 ++ TwitterAPI/__init__.py | 2 +- TwitterAPI/constants.py | 14 +++++++------- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index cb4190c..2af7aca 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -108,3 +108,5 @@ v2.4.7, 22 Oct 2017 -- Added endpoints for Ads API. v2.4.8, 03 Feb 2018 -- Merged RickRedSix patch. Added account activity endpoints. v2.4.9, 03 Feb 2018 -- Merged torufurukawa patch. Added webhook endpoint. + +v2.4.10, 11 Mar 2018 -- Merged torufurukawa patch. Added direct message endpoint. diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index d501c56..36736ba 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.4.9' +__version__ = '2.4.10' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 76fecea..fd4dbda 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -58,14 +58,14 @@ 'blocks/ids': ('GET', 'api'), 'blocks/list': ('GET', 'api'), - 'direct_messages': ('GET', 'api'), - 'direct_messages/destroy': ('POST', 'api'), + 'direct_messages': ('GET', 'api'), # deprecated + 'direct_messages/destroy': ('POST', 'api'), # deprecated 'direct_messages/events/new': ('POST', 'api'), 'direct_messages/events/list': ('GET', 'api'), 'direct_messages/events/show': ('GET', 'api'), - 'direct_messages/new': ('POST', 'api'), - 'direct_messages/sent': ('GET', 'api'), - 'direct_messages/show': ('GET', 'api'), + 'direct_messages/new': ('POST', 'api'), # deprecated + 'direct_messages/sent': ('GET', 'api'), # deprecated + 'direct_messages/show': ('GET', 'api'), # deprecated 'direct_messages/welcome_messages/new': ('POST', 'api'), 'direct_messages/welcome_messages/list': ('GET', 'api'), 'direct_messages/welcome_messages/show': ('GET', 'api'), @@ -151,8 +151,8 @@ 'statuses/show/:PARAM': ('GET', 'api'), # ID 'statuses/unretweet/:PARAM': ('POST', 'api'), # ID 'statuses/user_timeline': ('GET', 'api'), - 'statuses/update': ('POST', 'api'), # deprecated - 'statuses/update_with_media': ('POST', 'api'), + 'statuses/update': ('POST', 'api'), + 'statuses/update_with_media': ('POST', 'api'), # deprecated 'trends/available': ('GET', 'api'), 'trends/closest': ('GET', 'api'), From 0cbea3e90461113d3a109db5a0fc59e3d341ff2d Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 12 Mar 2018 08:24:07 -0400 Subject: [PATCH 005/139] Updated examples --- examples/api_test.py | 65 ------------------- .../{oauth_test.py => connect_using_login.py} | 2 +- ...w_to_connect.py => connect_using_oauth.py} | 28 ++++---- examples/delete_last_tweet.py | 3 +- examples/direct_message.py | 29 +++++++++ examples/logging.py | 31 +++++++++ examples/page_tweets.py | 18 ++--- examples/post_image.py | 30 --------- examples/post_tweet.py | 18 ++--- .../{sample_freq.py => sample_frequency.py} | 37 +++-------- examples/search_tweets.py | 18 ++--- examples/stream_tweets.py | 18 ++--- examples/unfollow_friends.py | 24 ++----- examples/{update_media.py => upload_image.py} | 23 ++----- examples/upload_tweet.py | 11 ++++ examples/upload_video.py | 20 ++---- 16 files changed, 131 insertions(+), 244 deletions(-) delete mode 100644 examples/api_test.py rename examples/{oauth_test.py => connect_using_login.py} (98%) rename examples/{how_to_connect.py => connect_using_oauth.py} (59%) create mode 100644 examples/direct_message.py create mode 100644 examples/logging.py delete mode 100644 examples/post_image.py rename examples/{sample_freq.py => sample_frequency.py} (66%) rename examples/{update_media.py => upload_image.py} (60%) create mode 100644 examples/upload_tweet.py diff --git a/examples/api_test.py b/examples/api_test.py deleted file mode 100644 index bbac44b..0000000 --- a/examples/api_test.py +++ /dev/null @@ -1,65 +0,0 @@ -import codecs -from datetime import datetime -import sys -from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRestPager - - -try: - # python 3 - sys.stdout = codecs.getwriter('utf8')(sys.stdout.buffer) -except: - # python 2 - sys.stdout = codecs.getwriter('utf8')(sys.stdout) - - -# SAVE YOUR APPLICATION CREDENTIALS IN TwitterAPI/credentials.txt. -o = TwitterOAuth.read_file() - -# Using OAuth1... -api = TwitterAPI(o.consumer_key, - o.consumer_secret, - o.access_token_key, - o.access_token_secret) - -# Using OAuth2... -#api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type="oAuth2") - - -TEST_NUMBER = 0 - - -try: - if TEST_NUMBER == 0: - - # VERIFY YOUR CREDS - r = api.request('account/verify_credentials') - print(r.text) - - if TEST_NUMBER == 1: - - # POST A TWEET - r = api.request('statuses/update', - {'status': 'the time is now %s' % datetime.now()}) - print(r.status_code) - - if TEST_NUMBER == 2: - - # GET 5 TWEETS CONTAINING 'ZZZ' - for item in api.request('search/tweets', {'q': 'zzz', 'count': 5}): - print(item['text'] if 'text' in item else item) - - if TEST_NUMBER == 3: - - # STREAM TWEETS FROM AROUND NYC - for item in api.request('statuses/filter', {'locations': '-74,40,-73,41'}): - print(item['text'] if 'text' in item else item) - - if TEST_NUMBER == 4: - - # GET TWEETS FROM THE PAST WEEK OR SO CONTAINING 'LOVE' - pager = TwitterRestPager(api, 'search/tweets', {'q': 'love'}) - for item in pager.get_iterator(): - print(item['text'] if 'text' in item else item) - -except Exception as e: - print(e) diff --git a/examples/oauth_test.py b/examples/connect_using_login.py similarity index 98% rename from examples/oauth_test.py rename to examples/connect_using_login.py index e694cc3..9b363d8 100644 --- a/examples/oauth_test.py +++ b/examples/connect_using_login.py @@ -43,4 +43,4 @@ access_token_key, access_token_secret) for item in api.request('statuses/filter', {'track': 'zzz'}): - print(item['text']) + print(item['text']) \ No newline at end of file diff --git a/examples/how_to_connect.py b/examples/connect_using_oauth.py similarity index 59% rename from examples/how_to_connect.py rename to examples/connect_using_oauth.py index c2b1606..7dd499d 100644 --- a/examples/how_to_connect.py +++ b/examples/connect_using_oauth.py @@ -1,31 +1,25 @@ from TwitterAPI import TwitterAPI - -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN_KEY = '' -ACCESS_TOKEN_SECRET = '' - - # If you are behind a firewall you may need to provide proxy server # authentication. proxy_url = None # Example: 'https://USERNAME:PASSWORD@PROXYSERVER:PORT' # Using OAuth 1.0 to authenticate you have access all Twitter endpoints. -# Using OAuth 2.0 to authenticate you lose access to user specific endpoints (ex. statuses/update), -# but you get higher rate limits. -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET, +# Using OAuth 2.0 to authenticate you lose access to user specific endpoints +# (ex. statuses/update and statuses/filter), but you increase your rate limits. +api = TwitterAPI(, + , + , + , auth_type='oAuth1', proxy_url=proxy_url) -#api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, auth_type='oAuth2', proxy_url=proxy_url) - +api = TwitterAPI(, + , + auth_type='oAuth2', + proxy_url=proxy_url) r = api.request('application/rate_limit_status') - # Print HTTP status code (=200 when no errors). print(r.status_code) @@ -34,4 +28,4 @@ # Parse the JSON response. j = r.response.json() -print(j['resources']['search']) +print(j['resources']['search']) \ No newline at end of file diff --git a/examples/delete_last_tweet.py b/examples/delete_last_tweet.py index cbf21b9..6347d92 100644 --- a/examples/delete_last_tweet.py +++ b/examples/delete_last_tweet.py @@ -8,7 +8,6 @@ , ) - class DeleteTweet(Thread): def __init__(self, tweet_id, count): @@ -32,4 +31,4 @@ def run(self): else: raise Exception(item) except Exception as e: - print('Stopping: %s' % str(e)) + print('Stopping: %s' % str(e)) \ No newline at end of file diff --git a/examples/direct_message.py b/examples/direct_message.py new file mode 100644 index 0000000..722cca8 --- /dev/null +++ b/examples/direct_message.py @@ -0,0 +1,29 @@ +from TwitterAPI import TwitterAPI +import json + +api = TwitterAPI(, + , + , + ) + +user_id = 334398732 +message_text = 'DM message 01' + +event = { + "event": { + "type": "message_create", + "message_create": { + "target": { + "recipient_id": user_id + }, + "message_data": { + "text": message_text + } + } + } +} + + +r = api.request('direct_messages/events/new', json.dumps(event)) + +print(r.status_code) \ No newline at end of file diff --git a/examples/logging.py b/examples/logging.py new file mode 100644 index 0000000..5a8782b --- /dev/null +++ b/examples/logging.py @@ -0,0 +1,31 @@ +from TwitterAPI import TwitterAPI +import logging + +# SET UP LOGGING TO FILE AND TO CONSOLE +formatter = logging.Formatter('%(levelname)s %(asctime)s %(message)s', + '%m/%d/%Y %I:%M:%S %p') +fh = logging.FileHandler('sample_freq.log') +fh.setFormatter(formatter) +ch = logging.StreamHandler() +ch.setFormatter(formatter) +logger = logging.getLogger() +logger.setLevel(logging.INFO) +logger.addHandler(fh) +logger.addHandler(ch) + +api = TwitterAPI(, + , + , + ) + +logging.info('START SAMPLE STREAM') + +try: + r = api.request('statuses/sample') + for item in r: + if 'text' in item: + print(item['text']) + elif: + logging.info('NOT A TWEET: %s' % item.text) +except Exception as e: + logging.warning('STOPPING: %s' % str(e)) \ No newline at end of file diff --git a/examples/page_tweets.py b/examples/page_tweets.py index 4ae9f01..1475db1 100644 --- a/examples/page_tweets.py +++ b/examples/page_tweets.py @@ -1,21 +1,13 @@ from TwitterAPI import TwitterAPI, TwitterRestPager - SEARCH_TERM = 'pizza' - -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN_KEY = '' -ACCESS_TOKEN_SECRET = '' - - -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET) +api = TwitterAPI(, + , + , + ) pager = TwitterRestPager(api, 'search/tweets', {'q': SEARCH_TERM}) for item in pager.get_iterator(): - print(item['text'] if 'text' in item else item) + print(item['text'] if 'text' in item else item) \ No newline at end of file diff --git a/examples/post_image.py b/examples/post_image.py deleted file mode 100644 index a71fc05..0000000 --- a/examples/post_image.py +++ /dev/null @@ -1,30 +0,0 @@ -''' -The method in this example for uploading image files is depricated. -See update_media.py for the recommeded way to tweet an image. -''' - -from TwitterAPI import TwitterAPI - - -TWEET_TEXT = 'some tweet text' -IMAGE_PATH = './some_image.png' - - -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN_KEY = '' -ACCESS_TOKEN_SECRET = '' - - -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET) - -file = open(IMAGE_PATH, 'rb') -data = file.read() -r = api.request('statuses/update_with_media', - {'status': TWEET_TEXT}, - {'media[]': data}) - -print('SUCCESS' if r.status_code == 200 else 'FAILURE') diff --git a/examples/post_tweet.py b/examples/post_tweet.py index 1956190..b14973e 100644 --- a/examples/post_tweet.py +++ b/examples/post_tweet.py @@ -1,20 +1,12 @@ from TwitterAPI import TwitterAPI - TWEET_TEXT = "Ce n'est pas un tweet tweet." - -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN_KEY = '' -ACCESS_TOKEN_SECRET = '' - - -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET) +api = TwitterAPI(, + , + , + ) r = api.request('statuses/update', {'status': TWEET_TEXT}) -print('SUCCESS' if r.status_code == 200 else 'FAILURE') +print('SUCCESS' if r.status_code == 200 else 'FAILURE') \ No newline at end of file diff --git a/examples/sample_freq.py b/examples/sample_frequency.py similarity index 66% rename from examples/sample_freq.py rename to examples/sample_frequency.py index 3d6ef31..ba84683 100644 --- a/examples/sample_freq.py +++ b/examples/sample_frequency.py @@ -1,27 +1,10 @@ -from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterConnectionError, TwitterRequestError +from TwitterAPI import TwitterAPI, TwitterConnectionError, TwitterRequestError import time -import logging - - -# SET UP LOGGING TO FILE AND TO CONSOLE -formatter = logging.Formatter('%(levelname)s %(asctime)s %(message)s', - '%m/%d/%Y %I:%M:%S %p') -fh = logging.FileHandler('sample_freq.log') -fh.setFormatter(formatter) -ch = logging.StreamHandler() -ch.setFormatter(formatter) -logger = logging.getLogger() -logger.setLevel(logging.INFO) -logger.addHandler(fh) -logger.addHandler(ch) - - -o = TwitterOAuth.read_file() -api = TwitterAPI(o.consumer_key, - o.consumer_secret, - o.access_token_key, - o.access_token_secret) +api = TwitterAPI(, + , + , + ) class Frequency: @@ -48,10 +31,8 @@ def update(self): self.interval_start = now self.interval_count = 0 - freq = Frequency() - while True: try: r = api.request('statuses/sample') @@ -59,15 +40,15 @@ def update(self): if 'text' in item: freq.update() elif 'limit' in item: - logging.info('TWEETS SKIPPED: %s' % item['limit']['track']) + print('TWEETS SKIPPED: %s' % item['limit']['track']) elif 'warning' in item: - logging.warning(item['warning']) + print(item['warning']) elif 'disconnect' in item: event = item['disconnect'] if event['code'] in [2,5,6,7]: # streaming connection rejected raise Exception(event) - logging.info('RE-CONNECTING: %s' % event) + print('RE-CONNECTING: %s' % event) break except TwitterRequestError as e: if e.status_code < 500: @@ -80,4 +61,4 @@ def update(self): break except Exception as e: print('STOPPED: %s %s' % (type(e), e)) - break + break \ No newline at end of file diff --git a/examples/search_tweets.py b/examples/search_tweets.py index adfc74f..a385284 100644 --- a/examples/search_tweets.py +++ b/examples/search_tweets.py @@ -1,23 +1,15 @@ from TwitterAPI import TwitterAPI - SEARCH_TERM = 'pizza' - -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN_KEY = '' -ACCESS_TOKEN_SECRET = '' - - -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET) +api = TwitterAPI(, + , + , + ) r = api.request('search/tweets', {'q': SEARCH_TERM}) for item in r: print(item['text'] if 'text' in item else item) -print('\nQUOTA: %s' % r.get_rest_quota()) +print('\nQUOTA: %s' % r.get_rest_quota()) \ No newline at end of file diff --git a/examples/stream_tweets.py b/examples/stream_tweets.py index ff2b0db..51da26b 100644 --- a/examples/stream_tweets.py +++ b/examples/stream_tweets.py @@ -1,21 +1,13 @@ from TwitterAPI import TwitterAPI - TRACK_TERM = 'pizza' - -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN_KEY = '' -ACCESS_TOKEN_SECRET = '' - - -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET) +api = TwitterAPI(, + , + , + ) r = api.request('statuses/filter', {'track': TRACK_TERM}) for item in r: - print(item['text'] if 'text' in item else item) + print(item['text'] if 'text' in item else item) \ No newline at end of file diff --git a/examples/unfollow_friends.py b/examples/unfollow_friends.py index 8c2838e..abc1ef7 100644 --- a/examples/unfollow_friends.py +++ b/examples/unfollow_friends.py @@ -1,30 +1,18 @@ -''' -Unfollow friends that do not follow you. -''' - +# Unfollow friends that do not follow you. from TwitterAPI import TwitterAPI - -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN = '' -ACCESS_TOKEN_SECRET = '' - - -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN, - ACCESS_TOKEN_SECRET) - +api = TwitterAPI(, + , + , + ) followers = set(id for id in api.request('followers/ids')) friends = set(id for id in api.request('friends/ids')) unfollow = set(followers) - set(friends) - for id in unfollow: r = api.request('friendships/destroy', {'user_id': id}) if r.status_code == 200: status = r.json() - print 'unfollowed %s' % status['screen_name'] + print 'unfollowed %s' % status['screen_name'] \ No newline at end of file diff --git a/examples/update_media.py b/examples/upload_image.py similarity index 60% rename from examples/update_media.py rename to examples/upload_image.py index e84d657..e68ccb4 100644 --- a/examples/update_media.py +++ b/examples/upload_image.py @@ -1,25 +1,14 @@ -''' -This example replaces post_image.py. It demonstrates a method for -posting an image, which now requires two API calls. -''' +# Post an image with tweet. Requires two API calls. from TwitterAPI import TwitterAPI - TWEET_TEXT = 'some tweet text' IMAGE_PATH = './some_image.png' - -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN_KEY = '' -ACCESS_TOKEN_SECRET = '' - - -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET) +api = TwitterAPI(, + , + , + ) # STEP 1 - upload image file = open(IMAGE_PATH, 'rb') @@ -32,4 +21,4 @@ media_id = r.json()['media_id'] r = api.request( 'statuses/update', {'status': TWEET_TEXT, 'media_ids': media_id}) - print('UPDATE STATUS SUCCESS' if r.status_code == 200 else 'UPDATE STATUS FAILURE') + print('UPDATE STATUS SUCCESS' if r.status_code == 200 else 'UPDATE STATUS FAILURE') \ No newline at end of file diff --git a/examples/upload_tweet.py b/examples/upload_tweet.py new file mode 100644 index 0000000..8eabb74 --- /dev/null +++ b/examples/upload_tweet.py @@ -0,0 +1,11 @@ +from TwitterAPI import TwitterAPI + +TWEET_TEXT = 'some tweet text' + +api = TwitterAPI(, + , + , + ) + +r = api.request('statuses/update', {'status':TWEET_TEXT}) +print(r.status_code) \ No newline at end of file diff --git a/examples/upload_video.py b/examples/upload_video.py index 41356bf..6912cc3 100644 --- a/examples/upload_video.py +++ b/examples/upload_video.py @@ -12,30 +12,22 @@ VIDEO_FILENAME = 'test.mp4' TWEET_TEXT = 'Video upload test' -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN_KEY = '' -ACCESS_TOKEN_SECRET = '' - - -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET) - +api = TwitterAPI(, + , + , + ) bytes_sent = 0 total_bytes = os.path.getsize(VIDEO_FILENAME) file = open(VIDEO_FILENAME, 'rb') - def check_status(r): + # EXIT PROGRAM WITH ERROR MESSAGE if r.status_code < 200 or r.status_code > 299: print(r.status_code) print(r.text) sys.exit(0) - r = api.request('media/upload', {'command':'INIT', 'media_type':'video/mp4', 'total_bytes':total_bytes}) check_status(r) @@ -54,4 +46,4 @@ def check_status(r): check_status(r) r = api.request('statuses/update', {'status':TWEET_TEXT, 'media_ids':media_id}) -check_status(r) +check_status(r) \ No newline at end of file From 2b2e51bf3a6810d62606f81ef22b63fae4799795 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 12 Mar 2018 17:57:29 -0400 Subject: [PATCH 006/139] Small changes to examples --- examples/connect_using_login.py | 9 ++------- examples/connect_using_oauth.py | 12 ++++++------ examples/delete_last_tweet.py | 2 +- examples/direct_message.py | 8 +++----- examples/logging.py | 2 +- examples/post_tweet.py | 3 +-- examples/sample_frequency.py | 4 ++++ examples/upload_image.py | 7 +++---- examples/upload_tweet.py | 2 +- examples/upload_video.py | 15 +++++++-------- 10 files changed, 29 insertions(+), 35 deletions(-) diff --git a/examples/connect_using_login.py b/examples/connect_using_login.py index 9b363d8..a3020d6 100644 --- a/examples/connect_using_login.py +++ b/examples/connect_using_login.py @@ -3,10 +3,8 @@ from urlparse import parse_qs from TwitterAPI import TwitterAPI - -consumer_key = '' -consumer_secret = '' - +consumer_key = +consumer_secret = # obtain request token oauth = OAuth1(consumer_key, consumer_secret) @@ -17,14 +15,12 @@ request_key = credentials.get('oauth_token')[0] request_secret = credentials.get('oauth_token_secret')[0] - # obtain authorization from resource owner print( 'Go here to authorize:\n https://api.twitter.com/oauth/authorize?oauth_token=%s' % request_key) verifier = raw_input('Enter your authorization code: ') - # obtain access token oauth = OAuth1(consumer_key, consumer_secret, @@ -36,7 +32,6 @@ access_token_key = credentials.get('oauth_token')[0] access_token_secret = credentials.get('oauth_token_secret')[0] - # access resource api = TwitterAPI(consumer_key, consumer_secret, diff --git a/examples/connect_using_oauth.py b/examples/connect_using_oauth.py index 7dd499d..46ef4cc 100644 --- a/examples/connect_using_oauth.py +++ b/examples/connect_using_oauth.py @@ -5,18 +5,18 @@ proxy_url = None # Example: 'https://USERNAME:PASSWORD@PROXYSERVER:PORT' # Using OAuth 1.0 to authenticate you have access all Twitter endpoints. -# Using OAuth 2.0 to authenticate you lose access to user specific endpoints -# (ex. statuses/update and statuses/filter), but you increase your rate limits. api = TwitterAPI(, , , , auth_type='oAuth1', proxy_url=proxy_url) -api = TwitterAPI(, - , - auth_type='oAuth2', - proxy_url=proxy_url) +# Using OAuth 2.0 to authenticate you lose access to user specific endpoints +# (ex. statuses/update and statuses/filter), but you increase your rate limits. +# api = TwitterAPI(, +# , +# auth_type='oAuth2', +# proxy_url=proxy_url) r = api.request('application/rate_limit_status') diff --git a/examples/delete_last_tweet.py b/examples/delete_last_tweet.py index 6347d92..3707079 100644 --- a/examples/delete_last_tweet.py +++ b/examples/delete_last_tweet.py @@ -17,7 +17,7 @@ def __init__(self, tweet_id, count): def run(self): r = api.request('statuses/destroy/:%d' % self.tweet_id) - print(self.count if r.status_code == 200 else r.text) + print(self.count if r.status_code == 200 else 'PROBLEM: ' + r.text) try: count = 0 diff --git a/examples/direct_message.py b/examples/direct_message.py index 722cca8..4e234aa 100644 --- a/examples/direct_message.py +++ b/examples/direct_message.py @@ -6,8 +6,8 @@ , ) -user_id = 334398732 -message_text = 'DM message 01' +user_id = +message_text = event = { "event": { @@ -23,7 +23,5 @@ } } - r = api.request('direct_messages/events/new', json.dumps(event)) - -print(r.status_code) \ No newline at end of file +print('SUCCESS' if r.status_code == 200 else 'PROBLEM: ' + r.text) \ No newline at end of file diff --git a/examples/logging.py b/examples/logging.py index 5a8782b..6bb29b2 100644 --- a/examples/logging.py +++ b/examples/logging.py @@ -4,7 +4,7 @@ # SET UP LOGGING TO FILE AND TO CONSOLE formatter = logging.Formatter('%(levelname)s %(asctime)s %(message)s', '%m/%d/%Y %I:%M:%S %p') -fh = logging.FileHandler('sample_freq.log') +fh = logging.FileHandler('logging.log') fh.setFormatter(formatter) ch = logging.StreamHandler() ch.setFormatter(formatter) diff --git a/examples/post_tweet.py b/examples/post_tweet.py index b14973e..d5977f2 100644 --- a/examples/post_tweet.py +++ b/examples/post_tweet.py @@ -8,5 +8,4 @@ ) r = api.request('statuses/update', {'status': TWEET_TEXT}) - -print('SUCCESS' if r.status_code == 200 else 'FAILURE') \ No newline at end of file +print('SUCCESS' if r.status_code == 200 else 'PROBLEM: ' + r.text) \ No newline at end of file diff --git a/examples/sample_frequency.py b/examples/sample_frequency.py index ba84683..8615f0d 100644 --- a/examples/sample_frequency.py +++ b/examples/sample_frequency.py @@ -33,6 +33,10 @@ def update(self): freq = Frequency() +# THIS DEMONSTRATES HOW TO HANDLE ALL TYPES OF STREAMING ERRORS. +# SO, THIS IS APPLICAPLE TO 'statuses/filter' AS WELL. ONLY WHEN +# APPROPRIATE, A DROPPED CONNECTION IS RE-ESTABLISHED. + while True: try: r = api.request('statuses/sample') diff --git a/examples/upload_image.py b/examples/upload_image.py index e68ccb4..11aa5a2 100644 --- a/examples/upload_image.py +++ b/examples/upload_image.py @@ -14,11 +14,10 @@ file = open(IMAGE_PATH, 'rb') data = file.read() r = api.request('media/upload', None, {'media': data}) -print('UPLOAD MEDIA SUCCESS' if r.status_code == 200 else 'UPLOAD MEDIA FAILURE') +print('UPLOAD MEDIA SUCCESS' if r.status_code == 200 else 'UPLOAD MEDIA FAILURE: ' + r.text) # STEP 2 - post tweet with a reference to uploaded image if r.status_code == 200: media_id = r.json()['media_id'] - r = api.request( - 'statuses/update', {'status': TWEET_TEXT, 'media_ids': media_id}) - print('UPDATE STATUS SUCCESS' if r.status_code == 200 else 'UPDATE STATUS FAILURE') \ No newline at end of file + r = api.request('statuses/update', {'status': TWEET_TEXT, 'media_ids': media_id}) + print('UPDATE STATUS SUCCESS' if r.status_code == 200 else 'UPDATE STATUS FAILURE: ' + r.text) \ No newline at end of file diff --git a/examples/upload_tweet.py b/examples/upload_tweet.py index 8eabb74..f9e7b45 100644 --- a/examples/upload_tweet.py +++ b/examples/upload_tweet.py @@ -8,4 +8,4 @@ ) r = api.request('statuses/update', {'status':TWEET_TEXT}) -print(r.status_code) \ No newline at end of file +print('SUCCESS' if r.status_code == 200 else 'PROBLEM: ' + r.text) \ No newline at end of file diff --git a/examples/upload_video.py b/examples/upload_video.py index 6912cc3..dc29c94 100644 --- a/examples/upload_video.py +++ b/examples/upload_video.py @@ -8,7 +8,6 @@ import os import sys - VIDEO_FILENAME = 'test.mp4' TWEET_TEXT = 'Video upload test' @@ -22,7 +21,7 @@ file = open(VIDEO_FILENAME, 'rb') def check_status(r): - # EXIT PROGRAM WITH ERROR MESSAGE + # EXIT PROGRAM WITH ERROR MESSAGE if r.status_code < 200 or r.status_code > 299: print(r.status_code) print(r.text) @@ -35,12 +34,12 @@ def check_status(r): segment_id = 0 while bytes_sent < total_bytes: - chunk = file.read(4*1024*1024) - r = api.request('media/upload', {'command':'APPEND', 'media_id':media_id, 'segment_index':segment_id}, {'media':chunk}) - check_status(r) - segment_id = segment_id + 1 - bytes_sent = file.tell() - print('[' + str(total_bytes) + ']', str(bytes_sent)) + chunk = file.read(4*1024*1024) + r = api.request('media/upload', {'command':'APPEND', 'media_id':media_id, 'segment_index':segment_id}, {'media':chunk}) + check_status(r) + segment_id = segment_id + 1 + bytes_sent = file.tell() + print('[' + str(total_bytes) + ']', str(bytes_sent)) r = api.request('media/upload', {'command':'FINALIZE', 'media_id':media_id}) check_status(r) From cfbfe8a3fc68c4a857ab7937d55ef98e77bea8b9 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Tue, 13 Mar 2018 10:33:48 -0400 Subject: [PATCH 007/139] Added more examples --- examples/lookup_tweet.py | 12 ++++++++++++ examples/lookup_user.py | 10 ++++++++++ 2 files changed, 22 insertions(+) create mode 100644 examples/lookup_tweet.py create mode 100644 examples/lookup_user.py diff --git a/examples/lookup_tweet.py b/examples/lookup_tweet.py new file mode 100644 index 0000000..1a8f950 --- /dev/null +++ b/examples/lookup_tweet.py @@ -0,0 +1,12 @@ +from TwitterAPI import TwitterAPI + +TWEET_ID = '964575983633252353' + +api = TwitterAPI(, + , + auth_type='oAuth2') + +r = api.request('statuses/show/:' + TWEET_ID) +tweet = r.json() +print(tweet['user']['screen_name'] + ':' + tweet['text'] if r.status_code == 200 + else 'PROBLEM: ' + r.text) \ No newline at end of file diff --git a/examples/lookup_user.py b/examples/lookup_user.py new file mode 100644 index 0000000..1b1a81a --- /dev/null +++ b/examples/lookup_user.py @@ -0,0 +1,10 @@ +from TwitterAPI import TwitterAPI + +SCREEN_NAME = 'TheTweetOfGod' + +api = TwitterAPI(, + , + auth_type='oAuth2') + +r = api.request('users/lookup', {'screen_name':SCREEN_NAME}) +print(r.json()[0]['id'] if r.status_code == 200 else 'PROBLEM: ' + r.text) From 1791dcd97b2f7c508df69bb6b529c74876c36729 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Tue, 13 Mar 2018 11:47:06 -0400 Subject: [PATCH 008/139] Added new example --- examples/dump_timeline.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 examples/dump_timeline.py diff --git a/examples/dump_timeline.py b/examples/dump_timeline.py new file mode 100644 index 0000000..f7dab55 --- /dev/null +++ b/examples/dump_timeline.py @@ -0,0 +1,23 @@ +# Print a user's timeline. This will get up to 3,200 tweets, which +# is the maximum the Twitter API allows. + +from TwitterAPI import TwitterAPI, TwitterRestPager + +SCREEN_NAME = 'TheTweetOfGod' + +api = TwitterAPI(, + , + auth_type='oAuth2') + +pager = TwitterRestPager(api, + 'statuses/user_timeline', + {'screen_name':SCREEN_NAME, 'count':200}) + +count = 0 +for item in pager.get_iterator(wait=3.5): + if 'text' in item: + count = count + 1 + print(count, item['text']) + elif 'message' in item: + print(item['message']) + break \ No newline at end of file From f781d3cfb6bbadd1a3c0087c8835ffff485a5d6f Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Tue, 13 Mar 2018 11:48:47 -0400 Subject: [PATCH 009/139] Added new example --- examples/dump_timeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/dump_timeline.py b/examples/dump_timeline.py index f7dab55..3bf6f4c 100644 --- a/examples/dump_timeline.py +++ b/examples/dump_timeline.py @@ -10,8 +10,8 @@ auth_type='oAuth2') pager = TwitterRestPager(api, - 'statuses/user_timeline', - {'screen_name':SCREEN_NAME, 'count':200}) + 'statuses/user_timeline', + {'screen_name':SCREEN_NAME, 'count':200}) count = 0 for item in pager.get_iterator(wait=3.5): From a1a57ffc5d3346dfc5e3aad7289557701e3ac436 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Thu, 15 Mar 2018 09:46:16 -0400 Subject: [PATCH 010/139] Renamed get_rest_quota() to get_quota() --- TwitterAPI/TwitterAPI.py | 2 +- examples/search_tweets.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 407e692..7f9cc6b 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -199,7 +199,7 @@ def __iter__(self): """ return self.get_iterator() - def get_rest_quota(self): + def get_quota(self): """Quota information in the REST-only response header. :returns: Dictionary of 'remaining' (count), 'limit' (count), 'reset' (time) diff --git a/examples/search_tweets.py b/examples/search_tweets.py index a385284..4067281 100644 --- a/examples/search_tweets.py +++ b/examples/search_tweets.py @@ -12,4 +12,4 @@ for item in r: print(item['text'] if 'text' in item else item) -print('\nQUOTA: %s' % r.get_rest_quota()) \ No newline at end of file +print('\nQUOTA: %s' % r.get_quota()) \ No newline at end of file From 76986c16f54cd264ee13e75299befaf5ea69d3b4 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Thu, 15 Mar 2018 09:56:51 -0400 Subject: [PATCH 011/139] Renamed TwitterRestPager to TwitterPager --- CHANGES.txt | 3 ++ TwitterAPI/TwitterRestPager.py | 98 ---------------------------------- TwitterAPI/__init__.py | 6 +-- docs/examples.rst | 2 +- docs/faulttolerance.rst | 4 +- docs/index.rst | 2 +- docs/paging.rst | 4 +- docs/twitterrestpager.rst | 6 --- examples/dump_timeline.py | 8 +-- examples/page_tweets.py | 4 +- tests/test_iterators.py | 8 +-- 11 files changed, 22 insertions(+), 123 deletions(-) delete mode 100644 TwitterAPI/TwitterRestPager.py delete mode 100644 docs/twitterrestpager.rst diff --git a/CHANGES.txt b/CHANGES.txt index 2af7aca..7373370 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -110,3 +110,6 @@ v2.4.8, 03 Feb 2018 -- Merged RickRedSix patch. Added account activity endpoi v2.4.9, 03 Feb 2018 -- Merged torufurukawa patch. Added webhook endpoint. v2.4.10, 11 Mar 2018 -- Merged torufurukawa patch. Added direct message endpoint. + +v2.5.0, 15 Mar 2018 -- Renamed get_rest_quota() to get_quota(). + Renamed TwitterRestPager to TwitterPager. diff --git a/TwitterAPI/TwitterRestPager.py b/TwitterAPI/TwitterRestPager.py deleted file mode 100644 index e7ce311..0000000 --- a/TwitterAPI/TwitterRestPager.py +++ /dev/null @@ -1,98 +0,0 @@ -__author__ = "geduldig" -__date__ = "June 8, 2013" -__license__ = "MIT" - - -from requests.exceptions import ConnectionError, ReadTimeout, SSLError -from requests.packages.urllib3.exceptions import ReadTimeoutError, ProtocolError -from .TwitterError import * -import requests -import time - - -class TwitterRestPager(object): - - """Continuous (stream-like) pagination of response from Twitter REST API resource. - - :param api: An authenticated TwitterAPI object - :param resource: String with the resource path (ex. search/tweets) - :param params: Dictionary of resource parameters - """ - - def __init__(self, api, resource, params=None): - self.api = api - self.resource = resource - self.params = params - - def get_iterator(self, wait=5, new_tweets=False): - """Iterate response from Twitter REST API resource. Resource is called - in a loop to retrieve consecutive pages of results. - - :param wait: Floating point number (default=5) of seconds wait between requests. - Depending on the resource, appropriate values are 5 or 60 seconds. - :param new_tweets: Boolean determining the search direction. - False (default) retrieves old results. - True retrieves current results. - - :returns: JSON objects containing statuses, errors or other return info. - :raises: TwitterRequestError - """ - elapsed = 0 - while True: - try: - # get one page of results - start = time.time() - r = self.api.request(self.resource, self.params) - it = r.get_iterator() - if new_tweets: - it = reversed(list(it)) - - # yield each item in the page - id = None - for item in it: - if 'id' in item: - id = item['id'] - if 'code' in item: - if item['code'] in [130, 131]: - # Twitter service error - raise TwitterConnectionError(item) - yield item - - # if a cursor is present, use it to get next page - # (otherwise, use id to get next page) - json = r.json() - cursor = -1 - if new_tweets and 'previous_cursor' in json: - cursor = json['previous_cursor'] - elif not new_tweets and 'next_cursor' in json: - cursor = json['next_cursor'] - - # bail when no more results - if cursor == 0: - break - elif cursor == -1 and not new_tweets and id is None: - break - - # sleep before getting another page of results - elapsed = time.time() - start - pause = wait - elapsed if elapsed < wait else 0 - time.sleep(pause) - - # waiting for new results to come in - if id is None: - continue - - # use either id or cursor to get a new page - if cursor != -1: - self.params['cursor'] = cursor - elif new_tweets: - self.params['since_id'] = str(id) - else: - self.params['max_id'] = str(id - 1) - - except TwitterRequestError as e: - if e.status_code < 500: - raise - continue - except TwitterConnectionError: - continue diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 36736ba..5f643f0 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.4.10' +__version__ = '2.5.0' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' @@ -16,7 +16,7 @@ from .TwitterAPI import TwitterAPI, TwitterResponse from .TwitterError import TwitterConnectionError, TwitterRequestError from .TwitterOAuth import TwitterOAuth - from .TwitterRestPager import TwitterRestPager + from .TwitterPager import TwitterPager except: pass @@ -26,5 +26,5 @@ 'TwitterConnectionError', 'TwitterRequestError', 'TwitterOAuth', - 'TwitterRestPager' + 'TwitterPager' ] diff --git a/docs/examples.rst b/docs/examples.rst index 723fb04..93d44af 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -26,7 +26,7 @@ Get your entire timeline .. code-block:: python - pager = TwitterRestPager(api, 'statuses/home_timeline', {'count':200}) + pager = TwitterPager(api, 'statuses/home_timeline', {'count':200}) for item in pager.get_iterator(wait=60): if 'text' in item: print item['text'] diff --git a/docs/faulttolerance.rst b/docs/faulttolerance.rst index 3bbc650..4416782 100644 --- a/docs/faulttolerance.rst +++ b/docs/faulttolerance.rst @@ -46,13 +46,13 @@ Last Week's Pages Requests for REST API endpoints can throw `TwitterRequestError <./twittererror.html>`_ and `TwitterConnectionError <./twittererror.html>`_. They do not, however, return "disconnect" messages. Twitter returns error messages for these endpoints with "message". Most of these errors require attention before re-trying the request, except those with codes of 130 or 131, which are internal server errors. -For making continuos REST API requests (i.e. paging), TwitterAPI provides `TwitterRestPager <./paging.html>`_. If you use this class to request tweets that have been posted back to one week old, for example, the class's iterator will handle both types of exceptions automatically. The iterator also handles "message" objects with 130 or 131 codes for you. Any other "message" object gets passed on for you to handle. +For making continuos REST API requests (i.e. paging), TwitterAPI provides `TwitterPager <./paging.html>`_. If you use this class to request tweets that have been posted back to one week old, for example, the class's iterator will handle both types of exceptions automatically. The iterator also handles "message" objects with 130 or 131 codes for you. Any other "message" object gets passed on for you to handle. One final consideration is the endpoint's rate limit, determinted by the endpoint and whether you authenticate with oAuth 1 or oAuth 2. By default, the iterator waits 5 seconds between requests. This is sufficient for 180 requests in 15 minutes, the rate limit for "search/tweets" with oAuth 1. You can do better with oAuth 2. It permits 450 requests every 15 minutes, or 1 request per 2 seconds. The example below sets the wait assuming oAuth 2 rate limits. .. code-block:: python - iterator = TwitterRestPager(api, 'search/tweets', {'q':'pizza'}).get_iterator(wait=2) + iterator = TwitterPager(api, 'search/tweets', {'q':'pizza'}).get_iterator(wait=2) try: for item in iterator: if 'text' in item: diff --git a/docs/index.rst b/docs/index.rst index 80112db..bd4af7a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -49,5 +49,5 @@ Optional: .. toctree:: :maxdepth: 2 - twitterrestpager.rst + twitterpager.rst twitteroauth.rst diff --git a/docs/paging.rst b/docs/paging.rst index 5e9f6bd..d48a5be 100644 --- a/docs/paging.rst +++ b/docs/paging.rst @@ -3,11 +3,11 @@ Paging Results Paging refers to getting successive batches of results. The Streaming API endpoints in a sense do this inherently. REST API endpoints will never return more than a specificied maximum number of results. When you request `search/tweets`, for example, by default you will get at most 20 tweets. You can increase that number to a maximum of 100. This is the page size. Twitter provides a `way `_ to get successive pages, so it is possible to get more than 100 tweets with `search/tweets`, just not in a single request. -If you don't want to implement paging yourself, you can use the `TwitterRestPager <./twitterrestpager.html>`_ helper class with any REST API endpoint that returns multiples of something. The following, for example, searches for all tweets containing 'pizza' that Twitter has stored -- about a week's worth. +If you don't want to implement paging yourself, you can use the `TwitterPager <./twitterpager.html>`_ helper class with any REST API endpoint that returns multiples of something. The following, for example, searches for all tweets containing 'pizza' that Twitter has stored -- about a week's worth. .. code-block:: python - r = TwitterRestPager(api, 'search/tweets', {'q':'pizza', 'count':100}) + r = TwitterPager(api, 'search/tweets', {'q':'pizza', 'count':100}) for item in r.get_iterator(): if 'text' in item: print item['text'] diff --git a/docs/twitterrestpager.rst b/docs/twitterrestpager.rst deleted file mode 100644 index 2fe221c..0000000 --- a/docs/twitterrestpager.rst +++ /dev/null @@ -1,6 +0,0 @@ -TwitterAPI.TwitterRestPager -=========================== - -.. automodule:: TwitterAPI.TwitterRestPager - :members: - :undoc-members: diff --git a/examples/dump_timeline.py b/examples/dump_timeline.py index 3bf6f4c..7f56230 100644 --- a/examples/dump_timeline.py +++ b/examples/dump_timeline.py @@ -1,7 +1,7 @@ # Print a user's timeline. This will get up to 3,200 tweets, which # is the maximum the Twitter API allows. -from TwitterAPI import TwitterAPI, TwitterRestPager +from TwitterAPI import TwitterAPI, TwitterPager SCREEN_NAME = 'TheTweetOfGod' @@ -9,9 +9,9 @@ , auth_type='oAuth2') -pager = TwitterRestPager(api, - 'statuses/user_timeline', - {'screen_name':SCREEN_NAME, 'count':200}) +pager = TwitterPager(api, + 'statuses/user_timeline', + {'screen_name':SCREEN_NAME, 'count':200}) count = 0 for item in pager.get_iterator(wait=3.5): diff --git a/examples/page_tweets.py b/examples/page_tweets.py index 1475db1..6a80931 100644 --- a/examples/page_tweets.py +++ b/examples/page_tweets.py @@ -1,4 +1,4 @@ -from TwitterAPI import TwitterAPI, TwitterRestPager +from TwitterAPI import TwitterAPI, TwitterPager SEARCH_TERM = 'pizza' @@ -7,7 +7,7 @@ , ) -pager = TwitterRestPager(api, 'search/tweets', {'q': SEARCH_TERM}) +pager = TwitterPager(api, 'search/tweets', {'q': SEARCH_TERM}) for item in pager.get_iterator(): print(item['text'] if 'text' in item else item) \ No newline at end of file diff --git a/tests/test_iterators.py b/tests/test_iterators.py index ed719a0..b85fe6c 100644 --- a/tests/test_iterators.py +++ b/tests/test_iterators.py @@ -35,10 +35,10 @@ def test_streaming_iterator(self): self.use_iterator(it) def test_paging_iterator(self): - pager = TwitterAPI.TwitterRestPager(self.api, - 'search/tweets', - {'q': 'pizza'}) - self.assertIsInstance(pager, TwitterAPI.TwitterRestPager) + pager = TwitterAPI.TwitterPager(self.api, + 'search/tweets', + {'q': 'pizza'}) + self.assertIsInstance(pager, TwitterAPI.TwitterPager) it = pager.get_iterator() self.use_iterator(it) From 1962291e122171d83f8d16f476300ea2e68dfebd Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Thu, 15 Mar 2018 09:57:34 -0400 Subject: [PATCH 012/139] Renamed TwitterRestPager to TwitterPager --- TwitterAPI/TwitterPager.py | 98 ++++++++++++++++++++++++++++++++++++++ docs/twitterpager.rst | 6 +++ 2 files changed, 104 insertions(+) create mode 100644 TwitterAPI/TwitterPager.py create mode 100644 docs/twitterpager.rst diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py new file mode 100644 index 0000000..4ff5ba6 --- /dev/null +++ b/TwitterAPI/TwitterPager.py @@ -0,0 +1,98 @@ +__author__ = "geduldig" +__date__ = "June 8, 2013" +__license__ = "MIT" + + +from requests.exceptions import ConnectionError, ReadTimeout, SSLError +from requests.packages.urllib3.exceptions import ReadTimeoutError, ProtocolError +from .TwitterError import * +import requests +import time + + +class TwitterPager(object): + + """Continuous (stream-like) pagination of response from Twitter REST API resource. + + :param api: An authenticated TwitterAPI object + :param resource: String with the resource path (ex. search/tweets) + :param params: Dictionary of resource parameters + """ + + def __init__(self, api, resource, params=None): + self.api = api + self.resource = resource + self.params = params + + def get_iterator(self, wait=5, new_tweets=False): + """Iterate response from Twitter REST API resource. Resource is called + in a loop to retrieve consecutive pages of results. + + :param wait: Floating point number (default=5) of seconds wait between requests. + Depending on the resource, appropriate values are 5 or 60 seconds. + :param new_tweets: Boolean determining the search direction. + False (default) retrieves old results. + True retrieves current results. + + :returns: JSON objects containing statuses, errors or other return info. + :raises: TwitterRequestError + """ + elapsed = 0 + while True: + try: + # get one page of results + start = time.time() + r = self.api.request(self.resource, self.params) + it = r.get_iterator() + if new_tweets: + it = reversed(list(it)) + + # yield each item in the page + id = None + for item in it: + if 'id' in item: + id = item['id'] + if 'code' in item: + if item['code'] in [130, 131]: + # Twitter service error + raise TwitterConnectionError(item) + yield item + + # if a cursor is present, use it to get next page + # (otherwise, use id to get next page) + json = r.json() + cursor = -1 + if new_tweets and 'previous_cursor' in json: + cursor = json['previous_cursor'] + elif not new_tweets and 'next_cursor' in json: + cursor = json['next_cursor'] + + # bail when no more results + if cursor == 0: + break + elif cursor == -1 and not new_tweets and id is None: + break + + # sleep before getting another page of results + elapsed = time.time() - start + pause = wait - elapsed if elapsed < wait else 0 + time.sleep(pause) + + # waiting for new results to come in + if id is None: + continue + + # use either id or cursor to get a new page + if cursor != -1: + self.params['cursor'] = cursor + elif new_tweets: + self.params['since_id'] = str(id) + else: + self.params['max_id'] = str(id - 1) + + except TwitterRequestError as e: + if e.status_code < 500: + raise + continue + except TwitterConnectionError: + continue diff --git a/docs/twitterpager.rst b/docs/twitterpager.rst new file mode 100644 index 0000000..ffd6103 --- /dev/null +++ b/docs/twitterpager.rst @@ -0,0 +1,6 @@ +TwitterAPI.TwitterPager +=========================== + +.. automodule:: TwitterAPI.TwitterPager + :members: + :undoc-members: From 317d5280e36a2006aa7217d303c2ec3904eea851 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 16 Mar 2018 22:31:16 -0400 Subject: [PATCH 013/139] 2.5 docs --- docs/README.rst | 37 +++++++++++++++++++++++++++++++++++++ docs/authentication.rst | 4 ++-- docs/conf.py | 4 ++-- docs/errors.rst | 6 +++--- docs/faulttolerance.rst | 13 ++++++------- docs/index.rst | 12 ++++++------ docs/paging.rst | 6 +++--- 7 files changed, 59 insertions(+), 23 deletions(-) create mode 100644 docs/README.rst diff --git a/docs/README.rst b/docs/README.rst new file mode 100644 index 0000000..b6bf8b8 --- /dev/null +++ b/docs/README.rst @@ -0,0 +1,37 @@ +Instructions for Generating Documentation with Sphinx +===================================================== + +Generate HTML Files +------------------- + +In a command shell:: + + cd TwitterAPI\docs + +Edit RST files. + +In a command shell:: + + make html + +Open in a browser:: + + TwitterAPI\docs\_build\html\index.html + +Repeat until happy + +Clean Up +-------- + +In a command shell:: + + rm -r TwitterAPI\docs\_build + +Publish to Git Repository +------------------------- + +In a command shell:: + + git add * + git commit -am '2.2.4 docs' + git push origin gh-pages diff --git a/docs/authentication.rst b/docs/authentication.rst index c9f6ad7..8af0979 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1,9 +1,9 @@ Authentication ============== -The first thing you need to do is create an application on `apps.twitter.com `_ and generate your oAuth keys. +Before you can do anything you must create an application on `apps.twitter.com `_ and generate oAuth keys. -Twitter supports both user and application authentication, known as oAuth 1 and oAuth 2, respectively. User authentication gives you access to all API endpoints, basically read and write persmission. Application authentication gives you access to just the read portion of the API -- so, no creating or destroying tweets. Application authentication also has elevated rate limits. +Twitter supports both user and application authentication, called oAuth 1 and oAuth 2, respectively. User authentication gives you access to all API endpoints, basically read and write persmission. It is also required in order to using the Streaming API. Application authentication gives you access to just the read portion of the API -- so, no creating or destroying tweets. Application authentication, however, has elevated rate limits. User Authentication ------------------- diff --git a/docs/conf.py b/docs/conf.py index 66e5e54..5f3d833 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -53,9 +53,9 @@ # built documents. # # The short X.Y version. -version = '2.3' +version = '2.5' # The full version, including alpha/beta/rc tags. -release = '2.3' +release = '2.5' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/errors.rst b/docs/errors.rst index 9d90d21..04471c0 100644 --- a/docs/errors.rst +++ b/docs/errors.rst @@ -1,7 +1,7 @@ Error Handling ============== -Besides tweet statuses, the REST API and Streaming API iterators may return error and other messages. It is up to the application to test what type of object has been returned. Message types are documented `here `_ and `here `_. +Besides tweet statuses, the REST API and Streaming API iterators can return errors and other messages. It is up to the application to test what type of object has been returned. Message types are documented `here `_ and `here `_. REST API Messages ----------------- @@ -20,7 +20,7 @@ REST API endpoints can return many more types of messages than Streaming API end Streaming API Messages ---------------------- -Streaming API endpoints return a variety of messages, most are not really errors. For example, a "limit" message contains the number of tweets missing from the stream. This happens when the number of tweets matching your filter exceeds a threshold set by Twitter. Other useful messages are "disconnect" and "delete". The pattern is similar to the one preceding: +Streaming API endpoints return a variety of messages. Some are not errors. For example, a "limit" message contains the number of tweets missing from the stream. This happens when the number of tweets matching your filter exceeds a threshold set by Twitter. Other useful messages are "disconnect" and "delete". .. code-block:: python @@ -34,4 +34,4 @@ Streaming API endpoints return a variety of messages, most are not really errors print 'disconnecting because %s' % item['disconnect']['reason'] break -Even if you are not interested in handling errors it is necessary to test that the object returned by an iterator is a valid tweet status before using the object. \ No newline at end of file +Even if you are not interested in handling errors it is necessary to test that the object returned by an iterator is a valid tweet status before using the object. Valid tweet objects have a 'text' property (or a 'full_text' property if it is an extended tweet). \ No newline at end of file diff --git a/docs/faulttolerance.rst b/docs/faulttolerance.rst index 4416782..f8059a3 100644 --- a/docs/faulttolerance.rst +++ b/docs/faulttolerance.rst @@ -53,10 +53,9 @@ One final consideration is the endpoint's rate limit, determinted by the endpoin .. code-block:: python iterator = TwitterPager(api, 'search/tweets', {'q':'pizza'}).get_iterator(wait=2) - try: - for item in iterator: - if 'text' in item: - print(item['text']) - elif 'message' in item: - # something needs to be fixed before re-connecting - raise Exception(item['message']) + for item in iterator: + if 'text' in item: + print(item['text']) + elif 'message' in item: + # something needs to be fixed before re-connecting + raise Exception(item['message']) \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index bd4af7a..439c4aa 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,7 +3,7 @@ Introduction *Minimal Python wrapper for Twitter's REST and Streaming APIs* -The principle behind TwitterAPI's design is to provide a single method for accessing the Twitter API. You can call the ``request`` method with *any* endpoint found on Twitter's `developer site `_, the complete reference for all endpoints. The benefits of a single-method approach are: less code for me to maintain, and just a single method for you to learn. Here is a quck example: +The principle behind TwitterAPI's design is to provide a single method for accessing the Twitter API. The ``request`` method allows one to call *any* endpoint found on Twitter's `developer site `_, the complete reference for all endpoints. The benefits of a single-method approach are: 1) less code for me to maintain, and 2) a single method for you to learn. Here is a quck example: .. code-block:: python @@ -12,16 +12,16 @@ The principle behind TwitterAPI's design is to provide a single method for acces r = api.request('search/tweets', {'q':'pizza'}) print r.status_code -If you want Twitter's entire response as one long string, containing tweets in this example, you would use ``r.text``. But, often an iterator is more useful: +Get Twitter's entire response as one long string containing tweets (in this example) by calling ``r.text``. More often an iterator is useful: .. code-block:: python for item in r.get_iterator(): print item['user']['screen_name'], item['text'] -The iterator returns decoded JSON objects. What makes the iterator very powerful is it works with both REST API and Streaming API endpoints. No syntax changes required; just supply any endpoint and parameters that are found on Twitter's dev site. +The iterator returns JSON objects. What makes the iterator very powerful is it works with both REST API and Streaming API endpoints. No syntax changes required -- supply any endpoint and parameters that are found on Twitter's developer site. -TwitterAPI is compatible with Python 2 and Python 3. It authenticates using either OAauth 1 or OAuth 2. It also supports web proxy server authentication. All with very little code change for you. +TwitterAPI is compatible with Python 2 and Python 3. It authenticates using either OAauth 1 or OAuth 2. It also supports web proxy server authentication. All this with minimal code change for you. Topics ====== @@ -30,10 +30,10 @@ Topics :maxdepth: 1 authentication.rst - errors.rst paging.rst - examples.rst + errors.rst faulttolerance.rst + examples.rst Modules ======= diff --git a/docs/paging.rst b/docs/paging.rst index d48a5be..7e53e47 100644 --- a/docs/paging.rst +++ b/docs/paging.rst @@ -1,9 +1,7 @@ Paging Results ============== -Paging refers to getting successive batches of results. The Streaming API endpoints in a sense do this inherently. REST API endpoints will never return more than a specificied maximum number of results. When you request `search/tweets`, for example, by default you will get at most 20 tweets. You can increase that number to a maximum of 100. This is the page size. Twitter provides a `way `_ to get successive pages, so it is possible to get more than 100 tweets with `search/tweets`, just not in a single request. - -If you don't want to implement paging yourself, you can use the `TwitterPager <./twitterpager.html>`_ helper class with any REST API endpoint that returns multiples of something. The following, for example, searches for all tweets containing 'pizza' that Twitter has stored -- about a week's worth. +Whether one is searching for tweets with `search/tweets` or downloading a user's timeline with `statuses/user_timeline` Twitter limits the number of tweets. So, in order to get more tweets, one must make successive requests and with each request skip the previously acquired tweets. This is done by specifying the tweet id from where to start. Twitter has a description `here `_. If you don't want to implement paging yourself, you can use the `TwitterPager <./twitterpager.html>`_ helper class with any REST API endpoint that returns multiples of something. The following, for example, searches for all tweets containing 'pizza' that Twitter has stored -- up to about a week's worth maximum. .. code-block:: python @@ -14,3 +12,5 @@ If you don't want to implement paging yourself, you can use the `TwitterPager <. elif 'message' in item and item['code'] == 88: print 'SUSPEND, RATE LIMIT EXCEEDED: %s\n' % item['message'] break + +By default there is a built-in wait time of 5 seconds between successive calls. This value is overridden with an argument to `get_iterator()`. See the documentation also to learn how to wait for new tweets. In other words, the iterator can be setup to poll for newer pages of tweets rather than older pages. \ No newline at end of file From 9c9454f0419cc0b8fd1c03b665fe94645c73ce2a Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sat, 17 Mar 2018 11:55:08 -0400 Subject: [PATCH 014/139] Updated docs/README.rst --- docs/README.rst | 39 +++++++++++++-------------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/docs/README.rst b/docs/README.rst index b6bf8b8..9dc69ae 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -1,37 +1,24 @@ Instructions for Generating Documentation with Sphinx ===================================================== -Generate HTML Files -------------------- - -In a command shell:: - - cd TwitterAPI\docs - -Edit RST files. - -In a command shell:: +MAKE NEW DOCS IN MASTER BRANCH:: + clone git@github.com:geduldig/TwitterAPI.git + cd TwitterAPI/docs make html -Open in a browser:: - - TwitterAPI\docs\_build\html\index.html - -Repeat until happy - -Clean Up --------- - -In a command shell:: +DOWNLOAD OLD DOCS FROM GH-PAGES BRANCH:: - rm -r TwitterAPI\docs\_build + cd TwitterAPI-docs + clone -b gh-pages git@github.com:geduldig/TwitterAPI.git -Publish to Git Repository -------------------------- +COPY NEW DOCS OVER OLD DOCS:: -In a command shell:: + cp TwitterAPI/docs/_build/html/*.* TwitterAPI-docs/TwitterAPI + cp -r TwitterAPI/docs/_build/html/* TwitterAPI-docs/TwitterAPI + rm -r TwitterAPI/docs/_build + +UPLOAD NEW DOCS TO GH-PAGES BRANCH::: - git add * - git commit -am '2.2.4 docs' + git commit -am "2.5 docs" git push origin gh-pages From ae6ceb0ffaa6544256143b68cdabdaedf03721de Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sat, 19 May 2018 17:27:20 -0400 Subject: [PATCH 015/139] Added support for both HTTP and HTTPS proxies --- CHANGES.txt | 2 ++ TwitterAPI/TwitterAPI.py | 10 ++++++++-- TwitterAPI/__init__.py | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 7373370..5029406 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -113,3 +113,5 @@ v2.4.10, 11 Mar 2018 -- Merged torufurukawa patch. Added direct message endpoi v2.5.0, 15 Mar 2018 -- Renamed get_rest_quota() to get_quota(). Renamed TwitterRestPager to TwitterPager. + +v2.5.1, 19 May 2018 -- Added support for both HTTP and HTTPS proxies. diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 7f9cc6b..ba6ee93 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -26,7 +26,8 @@ class TwitterAPI(object): :param access_token_key: Twitter application access token key :param access_token_secret: Twitter application access token secret :param auth_type: "oAuth1" (default) or "oAuth2" - :param proxy_url: HTTPS proxy URL (ex. "https://USER:PASSWORD@SERVER:PORT") + :param proxy_url: HTTPS proxy URL string (ex. "https://USER:PASSWORD@SERVER:PORT"), + or dict of URLs (ex. {'http':'http://SERVER', 'https':'https://SERVER'}) """ def __init__( @@ -38,7 +39,12 @@ def __init__( auth_type='oAuth1', proxy_url=None): """Initialize with your Twitter application credentials""" - self.proxies = {'https': proxy_url} if proxy_url else None + if isinstance(proxy_url, dict): + self.proxies = proxy_url + elif proxy_url is not None: + self.proxies = {'https': proxy_url} + else: + self.proxies = None if auth_type == 'oAuth1': if not all([consumer_key, consumer_secret, access_token_key, access_token_secret]): raise Exception('Missing authentication parameter') diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 5f643f0..88aec62 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.5.0' +__version__ = '2.5.1' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 2564ced477704c70f678c85c2a735cdba6a108f9 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sat, 19 May 2018 18:36:27 -0400 Subject: [PATCH 016/139] Added endpoints for Premium Search API --- CHANGES.txt | 2 ++ README.rst | 2 ++ TODO.txt | 2 ++ TwitterAPI/__init__.py | 2 +- TwitterAPI/constants.py | 11 +++++++++-- 5 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 5029406..831060d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -115,3 +115,5 @@ v2.5.0, 15 Mar 2018 -- Renamed get_rest_quota() to get_quota(). Renamed TwitterRestPager to TwitterPager. v2.5.1, 19 May 2018 -- Added support for both HTTP and HTTPS proxies. + +v2.5.2, 19 May 2018 -- Added endpoints for Premium Search API. diff --git a/README.rst b/README.rst index 211d9a0..ba1c8eb 100755 --- a/README.rst +++ b/README.rst @@ -12,6 +12,8 @@ TwitterAPI is a Python package for accessing Twitter's REST APIs and Streaming APIs. It supports OAuth 1.0 and OAuth 2.0 authentication. And, it works with the latest Python versions in both 2.x and 3.x branches. +REST APIs that are supported are: Public API, Collections API, Curator API, Ads API, Webhook API, Premium Search API. + Installation ------------ From the command line:: diff --git a/TODO.txt b/TODO.txt index 5ee980e..dc023b8 100644 --- a/TODO.txt +++ b/TODO.txt @@ -2,3 +2,5 @@ Build test suite with unittest. (If anyone wants to contribute, would be much appreciated.) Test Ads API endpoints. + +Add and test Enterprise Search API endpoints. diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 88aec62..453f462 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.5.1' +__version__ = '2.5.2' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index fd4dbda..6faed67 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -216,5 +216,12 @@ 'account_activity/webhooks': ('POST', 'api'), 'account_activity/webhooks/:PARAM': ('DELETE', 'api'), # WEBHOOK ID 'account_activity/webhooks/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME - 'account_activity/webhooks/:PARAM/subscriptions/list': ('GET', 'api') # ENVIRONMENT NAME -} + 'account_activity/webhooks/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME + + # PREMIUM SEARCH API + + 'tweets/search/30day/:PARAM': ('GET', 'api'), # LABEL + 'tweets/search/30day/:PARAM/counts': ('GET', 'api'), # LABEL + 'tweets/search/fullarchive/:PARAM': ('GET', 'api'), # LABEL + 'tweets/search/fullarchive/:PARAM/counts': ('GET', 'api') # LABEL +} \ No newline at end of file From 3c6226d1be6cda7454c5e268f6f029c7ba803c8d Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sun, 20 May 2018 10:30:15 -0400 Subject: [PATCH 017/139] Added 'results' to iterator and a new Premium Search example --- CHANGES.txt | 2 ++ TwitterAPI/TwitterAPI.py | 2 ++ TwitterAPI/__init__.py | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 831060d..b7ef898 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -117,3 +117,5 @@ v2.5.0, 15 Mar 2018 -- Renamed get_rest_quota() to get_quota(). v2.5.1, 19 May 2018 -- Added support for both HTTP and HTTPS proxies. v2.5.2, 19 May 2018 -- Added endpoints for Premium Search API. + +v2.5.3, 19 May 2018 -- Added 'results' to iterator and a new Premium Search example. diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index ba6ee93..31148ca 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -244,6 +244,8 @@ def __init__(self, response): self.results = resp['users'] elif 'ids' in resp: self.results = resp['ids'] + elif 'results' in resp: + self.results = resp['results'] elif 'data' in resp and not isinstance(resp['data'], dict): self.results = resp['data'] elif hasattr(resp, '__iter__') and not isinstance(resp, dict): diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 453f462..9686951 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.5.2' +__version__ = '2.5.3' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 62410d10f04d825ff55a9c46ab8e16d1c879eadf Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sun, 20 May 2018 10:32:16 -0400 Subject: [PATCH 018/139] Forgot to add premium_search.py to git --- examples/premium_search.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 examples/premium_search.py diff --git a/examples/premium_search.py b/examples/premium_search.py new file mode 100644 index 0000000..3847a66 --- /dev/null +++ b/examples/premium_search.py @@ -0,0 +1,17 @@ +from TwitterAPI import TwitterAPI + + +SEARCH_TERM = 'pizza' +PRODUCT = '30day' +LABEL = 'your label' + +api = TwitterAPI(, + , + , + ) + +r = api.request('tweets/search/%s/:%s' % (PRODUCT, LABEL), + {'query':SEARCH_TERM}) + +for item in r: + print(item['text'] if 'text' in item else item) \ No newline at end of file From 960905a05f4dd5d27ac300282466717f026a0e03 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 25 May 2018 22:22:25 -0400 Subject: [PATCH 019/139] Modified TwitterPager to support Premium Search's cursor --- CHANGES.txt | 2 ++ TwitterAPI/TwitterPager.py | 12 ++++++++++-- TwitterAPI/__init__.py | 2 +- examples/premium_search.py | 5 ++--- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index b7ef898..89b500b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -119,3 +119,5 @@ v2.5.1, 19 May 2018 -- Added support for both HTTP and HTTPS proxies. v2.5.2, 19 May 2018 -- Added endpoints for Premium Search API. v2.5.3, 19 May 2018 -- Added 'results' to iterator and a new Premium Search example. + +v2.5.4, 25 May 2018 -- Modified TwitterPager to support Premium Search's cursor. diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py index 4ff5ba6..b3e58ee 100644 --- a/TwitterAPI/TwitterPager.py +++ b/TwitterAPI/TwitterPager.py @@ -13,6 +13,7 @@ class TwitterPager(object): """Continuous (stream-like) pagination of response from Twitter REST API resource. + In addition to Public API endpoints, supports Premium Search API. :param api: An authenticated TwitterAPI object :param resource: String with the resource path (ex. search/tweets) @@ -59,13 +60,20 @@ def get_iterator(self, wait=5, new_tweets=False): yield item # if a cursor is present, use it to get next page - # (otherwise, use id to get next page) + # otherwise, use id to get next page json = r.json() cursor = -1 if new_tweets and 'previous_cursor' in json: cursor = json['previous_cursor'] + cursor_param = 'cursor' elif not new_tweets and 'next_cursor' in json: cursor = json['next_cursor'] + cursor_param = 'cursor' + elif not new_tweets and 'next' in json: + # 'next' is used by Premium Search, so only + # works searching back in time + cursor = json['next'] + cursor_param = 'next' # bail when no more results if cursor == 0: @@ -84,7 +92,7 @@ def get_iterator(self, wait=5, new_tweets=False): # use either id or cursor to get a new page if cursor != -1: - self.params['cursor'] = cursor + self.params[cursor_param] = cursor elif new_tweets: self.params['since_id'] = str(id) else: diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 9686951..0cc27f6 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.5.3' +__version__ = '2.5.4' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/examples/premium_search.py b/examples/premium_search.py index 3847a66..f5426ff 100644 --- a/examples/premium_search.py +++ b/examples/premium_search.py @@ -1,6 +1,5 @@ from TwitterAPI import TwitterAPI - SEARCH_TERM = 'pizza' PRODUCT = '30day' LABEL = 'your label' @@ -11,7 +10,7 @@ ) r = api.request('tweets/search/%s/:%s' % (PRODUCT, LABEL), - {'query':SEARCH_TERM}) + {'query':SEARCH_TERM}) for item in r: - print(item['text'] if 'text' in item else item) \ No newline at end of file + print(item['text'] if 'text' in item else item) From 8749b342098061c57117ca5b36a249d7ecb2dc61 Mon Sep 17 00:00:00 2001 From: x0139 Date: Mon, 17 Sep 2018 22:50:59 +0200 Subject: [PATCH 020/139] add Typing indicator endpoint --- .gitignore | 2 ++ TwitterAPI/constants.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 58e178d..226971a 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,5 @@ nosetests.xml .mr.developer.cfg .project .pydevproject +.idea/* +env/* \ No newline at end of file diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 6faed67..2c74266 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -74,7 +74,8 @@ 'direct_messages/welcome_messages/rules/list': ('GET', 'api'), 'direct_messages/welcome_messages/rules/show': ('GET', 'api'), 'direct_messages/welcome_messages/rules/destroy': ('DELETE', 'api'), - + 'direct_messages/indicate_typing': ('POST', 'api'), + 'favorites/create': ('POST', 'api'), 'favorites/destroy': ('POST', 'api'), 'favorites/list': ('GET', 'api'), From 6dcc40e03e8dcdcd3db80a88479a4a122d67a038 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sat, 6 Oct 2018 09:24:38 -0400 Subject: [PATCH 021/139] Added new endpoint --- CHANGES.txt | 3 +++ TwitterAPI/TwitterPager.py | 4 ++-- TwitterAPI/__init__.py | 2 +- TwitterAPI/constants.py | 5 +++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 89b500b..623cba2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -121,3 +121,6 @@ v2.5.2, 19 May 2018 -- Added endpoints for Premium Search API. v2.5.3, 19 May 2018 -- Added 'results' to iterator and a new Premium Search example. v2.5.4, 25 May 2018 -- Modified TwitterPager to support Premium Search's cursor. + +v2.5.5, 25 May 2018 -- Merged x0139 patch. Added typing indicator endpoint. + Added DM destroy endpoint. diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py index b3e58ee..4f3a191 100644 --- a/TwitterAPI/TwitterPager.py +++ b/TwitterAPI/TwitterPager.py @@ -70,8 +70,8 @@ def get_iterator(self, wait=5, new_tweets=False): cursor = json['next_cursor'] cursor_param = 'cursor' elif not new_tweets and 'next' in json: - # 'next' is used by Premium Search, so only - # works searching back in time + # 'next' is used by Premium Search, so only + # works searching back in time cursor = json['next'] cursor_param = 'next' diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 0cc27f6..7bbae1a 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.5.4' +__version__ = '2.5.5' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 2c74266..0ef31b8 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -60,9 +60,11 @@ 'direct_messages': ('GET', 'api'), # deprecated 'direct_messages/destroy': ('POST', 'api'), # deprecated + 'direct_messages/events/destroy': ('DELETE', 'api'), 'direct_messages/events/new': ('POST', 'api'), 'direct_messages/events/list': ('GET', 'api'), 'direct_messages/events/show': ('GET', 'api'), + 'direct_messages/indicate_typing': ('POST', 'api'), 'direct_messages/new': ('POST', 'api'), # deprecated 'direct_messages/sent': ('GET', 'api'), # deprecated 'direct_messages/show': ('GET', 'api'), # deprecated @@ -74,7 +76,6 @@ 'direct_messages/welcome_messages/rules/list': ('GET', 'api'), 'direct_messages/welcome_messages/rules/show': ('GET', 'api'), 'direct_messages/welcome_messages/rules/destroy': ('DELETE', 'api'), - 'direct_messages/indicate_typing': ('POST', 'api'), 'favorites/create': ('POST', 'api'), 'favorites/destroy': ('POST', 'api'), @@ -225,4 +226,4 @@ 'tweets/search/30day/:PARAM/counts': ('GET', 'api'), # LABEL 'tweets/search/fullarchive/:PARAM': ('GET', 'api'), # LABEL 'tweets/search/fullarchive/:PARAM/counts': ('GET', 'api') # LABEL -} \ No newline at end of file +} From d14d6838225f89ea2effa9173e021f1461437f00 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Tue, 9 Oct 2018 13:49:00 -0400 Subject: [PATCH 022/139] Fixed TwitterPager --- CHANGES.txt | 4 +++- TwitterAPI/TwitterPager.py | 17 +++++++++-------- TwitterAPI/__init__.py | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 623cba2..f1d21de 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -122,5 +122,7 @@ v2.5.3, 19 May 2018 -- Added 'results' to iterator and a new Premium Search e v2.5.4, 25 May 2018 -- Modified TwitterPager to support Premium Search's cursor. -v2.5.5, 25 May 2018 -- Merged x0139 patch. Added typing indicator endpoint. +v2.5.5, 07 Oct 2018 -- Merged x0139 patch. Added typing indicator endpoint. Added DM destroy endpoint. + +v2.5.6, 09 Oct 2018 -- Fixed TwitterPager. diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py index 4f3a191..f64d1e5 100644 --- a/TwitterAPI/TwitterPager.py +++ b/TwitterAPI/TwitterPager.py @@ -86,17 +86,18 @@ def get_iterator(self, wait=5, new_tweets=False): pause = wait - elapsed if elapsed < wait else 0 time.sleep(pause) - # waiting for new results to come in - if id is None: - continue - - # use either id or cursor to get a new page + # waiting for new results to come in... + # get a page with cursor if present, or with id if not + # a Premium search (i.e. 'query' is not a parameter) if cursor != -1: self.params[cursor_param] = cursor - elif new_tweets: - self.params['since_id'] = str(id) + elif id is not None and 'query' not in self.params: + if new_tweets: + self.params['since_id'] = str(id) + else: + self.params['max_id'] = str(id - 1) else: - self.params['max_id'] = str(id - 1) + continue except TwitterRequestError as e: if e.status_code < 500: diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 7bbae1a..3a05a40 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.5.5' +__version__ = '2.5.6' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 157ddc5bbdc4797c3ac1541a885c773c015aad90 Mon Sep 17 00:00:00 2001 From: Rick Smith Date: Tue, 16 Oct 2018 16:32:20 +0100 Subject: [PATCH 023/139] Updating endpoints to released account activity API --- TwitterAPI/constants.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 7cb9113..b4adc41 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -50,21 +50,22 @@ 'account/update_profile_image': ('POST', 'api'), 'account/verify_credentials': ('GET', 'api'), - # NEW ACCOUNT ACTIVITY WEBHOOK API (ALL EVENTS) (BETA) - - 'account_activity/all/:PARAM/webhooks': ('POST', 'api'), - 'account_activity/all/:PARAM/subscriptions': ('POST', 'api'), - 'account_activity/all/webhooks': ('GET', 'api'), - 'account_activity/all/count': ('GET', 'api'), - 'account_activity/all/:PARAM/subscriptions/all': ('GET', 'api'), - 'account_activity/all/:PARAM/subscriptions/all/list': ('GET', 'api'), - 'account_activity/all/:PARAM/subscriptions': ('DELETE', 'api'), + # NEW ACCOUNT ACTIVITY WEBHOOK API + 'account_activity/all/:PARAM/webhooks': ('POST', 'api'), # ENV_NAME + 'account_activity/all/webhooks': ('GET', 'api'), + + 'account_activity/all/:PARAM/subscriptions': ('POST', 'api'), # ENV_NAME + 'account_activity/all/subscriptions/count': ('GET', 'api'), + 'account_activity/all/:PARAM/subscriptions': ('GET', 'api'), # ENV_NAME + 'account_activity/all/:PARAM/subscriptions/list': ('GET', 'api'), # ENV_NAME - # NEW ACCOUNT ACTIVITY WEBHOOK API (DM) (BETA) - 'account_activity/webhooks': ('POST', 'api'), - 'account_activity/webhooks/:PARAM': ('DELETE', 'api'), - 'account_activity/webhooks/:PARAM/subscriptions': ('POST', 'api'), - 'account_activity/webhooks/:PARAM/subscriptions/list': ('GET', 'api'), + # ENTERPRISE ACCOUNT ACTIVITY WEBHOOK API + 'account_activity/webhooks': ('POST', 'api'), + 'account_activity/webhooks/:PARAM': ('DELETE', 'api'), #WEBHOOK_ID + 'account_activity/webhooks/:PARAM/subscriptions': ('POST', 'api'), #WEBHOOK_ID + 'account_activity/webhooks/count': ('GET', 'api'), + 'account_activity/webhooks/:PARAM/subscriptions/all/list': ('GET', 'api'), #WEBHOOK_ID + 'account_activity/webhooks/:PARAM/subscriptions/all': ('POST', 'api'), #WEBHOOK_ID 'application/rate_limit_status': ('GET', 'api'), From 8f3b2210c8b9abd585e1fa2b88fb25f9233dfbe7 Mon Sep 17 00:00:00 2001 From: Randy Ram Date: Fri, 14 Dec 2018 17:01:30 -0400 Subject: [PATCH 024/139] Added endpoint for premium accounts to view list of subscriptions --- TwitterAPI/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 0ef31b8..8a1d5c4 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -210,6 +210,7 @@ 'account_activity/all/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME 'account_activity/all/:PARAM/subscriptions/all': ('GET', 'api'), # ENVIRONMENT NAME 'account_activity/all/:PARAM/subscriptions/all/list': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME 'account_activity/all/:PARAM/subscriptions': ('DELETE', 'api'), # ENVIRONMENT NAME 'account_activity/all/:PARAM/webhooks': ('POST', 'api'), # ENVIRONMENT NAME 'account_activity/all/:PARAM/webhooks/:PARAM': ('DELETE', 'api'), # ENVIRONMENT NAME, WEBHOOK ID From f4a99ff2bd3962424740bf5dd6915d8aa9258daf Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sat, 15 Dec 2018 09:06:13 -0500 Subject: [PATCH 025/139] Merged HyperManTT patch --- CHANGES.txt | 2 ++ TwitterAPI/__init__.py | 2 +- TwitterAPI/constants.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index f1d21de..e9f7612 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -126,3 +126,5 @@ v2.5.5, 07 Oct 2018 -- Merged x0139 patch. Added typing indicator endpoint. Added DM destroy endpoint. v2.5.6, 09 Oct 2018 -- Fixed TwitterPager. + +v2.5.7, 09 Oct 2018 -- Merged HyperManTT path. Added subscripts/list endpoint. diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 3a05a40..7be589e 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.5.6' +__version__ = '2.5.7' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 8a1d5c4..22f5acf 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -208,10 +208,10 @@ # NEW ACCOUNT ACTIVITY WEBHOOK API (BETA) 'account_activity/all/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions': ('DELETE', 'api'), # ENVIRONMENT NAME 'account_activity/all/:PARAM/subscriptions/all': ('GET', 'api'), # ENVIRONMENT NAME 'account_activity/all/:PARAM/subscriptions/all/list': ('GET', 'api'), # ENVIRONMENT NAME 'account_activity/all/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/subscriptions': ('DELETE', 'api'), # ENVIRONMENT NAME 'account_activity/all/:PARAM/webhooks': ('POST', 'api'), # ENVIRONMENT NAME 'account_activity/all/:PARAM/webhooks/:PARAM': ('DELETE', 'api'), # ENVIRONMENT NAME, WEBHOOK ID 'account_activity/all/count': ('GET', 'api'), From 43aee51d5f47468e50c42a8fd7a7135fdd622da4 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Tue, 25 Dec 2018 10:59:38 -0500 Subject: [PATCH 026/139] Fixed bug in TwitterPager --- CHANGES.txt | 4 +++- TwitterAPI/TwitterPager.py | 8 +++++--- TwitterAPI/__init__.py | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index e9f7612..fcc0376 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -127,4 +127,6 @@ v2.5.5, 07 Oct 2018 -- Merged x0139 patch. Added typing indicator endpoint. v2.5.6, 09 Oct 2018 -- Fixed TwitterPager. -v2.5.7, 09 Oct 2018 -- Merged HyperManTT path. Added subscripts/list endpoint. +v2.5.7, 15 Dec 2018 -- Merged HyperManTT path. Added subscripts/list endpoint. + +v2.5.8, 25 Dec 2018 -- Fixed bug in TwitterPager, repeating results. diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py index f64d1e5..2a9be78 100644 --- a/TwitterAPI/TwitterPager.py +++ b/TwitterAPI/TwitterPager.py @@ -75,10 +75,12 @@ def get_iterator(self, wait=5, new_tweets=False): cursor = json['next'] cursor_param = 'next' + is_premium_search = 'query' in self.params + # bail when no more results if cursor == 0: break - elif cursor == -1 and not new_tweets and id is None: + elif cursor == -1 and is_premium_search: break # sleep before getting another page of results @@ -91,7 +93,7 @@ def get_iterator(self, wait=5, new_tweets=False): # a Premium search (i.e. 'query' is not a parameter) if cursor != -1: self.params[cursor_param] = cursor - elif id is not None and 'query' not in self.params: + elif id is not None and not is_premium_search: if new_tweets: self.params['since_id'] = str(id) else: @@ -104,4 +106,4 @@ def get_iterator(self, wait=5, new_tweets=False): raise continue except TwitterConnectionError: - continue + continue \ No newline at end of file diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 7be589e..3d98e56 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.5.7' +__version__ = '2.5.8' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 1ad801358e73b859693c35c43990b553508e1926 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 14 Jan 2019 23:09:01 -0500 Subject: [PATCH 027/139] Fixed bug in TwitterPager, stalling --- CHANGES.txt | 2 ++ TwitterAPI/TwitterPager.py | 4 ++++ TwitterAPI/__init__.py | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index fcc0376..acb20fc 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -130,3 +130,5 @@ v2.5.6, 09 Oct 2018 -- Fixed TwitterPager. v2.5.7, 15 Dec 2018 -- Merged HyperManTT path. Added subscripts/list endpoint. v2.5.8, 25 Dec 2018 -- Fixed bug in TwitterPager, repeating results. + +v2.5.9, 14 Jan 2019 -- Fixed bug in TwitterPager, stalling but not exiting when no more results. diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py index 2a9be78..4787199 100644 --- a/TwitterAPI/TwitterPager.py +++ b/TwitterAPI/TwitterPager.py @@ -49,8 +49,10 @@ def get_iterator(self, wait=5, new_tweets=False): it = reversed(list(it)) # yield each item in the page + item_count = 0 id = None for item in it: + item_count = item_count + 1 if 'id' in item: id = item['id'] if 'code' in item: @@ -82,6 +84,8 @@ def get_iterator(self, wait=5, new_tweets=False): break elif cursor == -1 and is_premium_search: break + elif not new_tweets and item_count == 0: + break # sleep before getting another page of results elapsed = time.time() - start diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 3d98e56..2840391 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.5.8' +__version__ = '2.5.9' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 0f9046f942a8181da7e39b0a6e93b49a2baf4a87 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Wed, 16 Jan 2019 07:40:44 -0500 Subject: [PATCH 028/139] Fixed bug in TwitterPager, stalling... --- CHANGES.txt | 1 + TwitterAPI/BearerAuth.py | 5 +++-- TwitterAPI/TwitterAPI.py | 25 +++++++++++++++++++------ TwitterAPI/constants.py | 7 ------- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index acb20fc..52c258e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -132,3 +132,4 @@ v2.5.7, 15 Dec 2018 -- Merged HyperManTT path. Added subscripts/list endpoint v2.5.8, 25 Dec 2018 -- Fixed bug in TwitterPager, repeating results. v2.5.9, 14 Jan 2019 -- Fixed bug in TwitterPager, stalling but not exiting when no more results. + Moved USER_AGENT and TIMEOUT constants to TwitterAPI class static variables. diff --git a/TwitterAPI/BearerAuth.py b/TwitterAPI/BearerAuth.py index 335c805..be487bc 100644 --- a/TwitterAPI/BearerAuth.py +++ b/TwitterAPI/BearerAuth.py @@ -21,10 +21,11 @@ class BearerAuth(requests.auth.AuthBase): :param proxies: Dictionary of proxy URLs (see documentation for python-requests). """ - def __init__(self, consumer_key, consumer_secret, proxies=None): + def __init__(self, consumer_key, consumer_secret, proxies=None, user_agent=None): self._consumer_key = consumer_key self._consumer_secret = consumer_secret self.proxies = proxies + self.user_agent = user_agent self._bearer_token = self._get_access_token() def _get_access_token(self): @@ -36,7 +37,7 @@ def _get_access_token(self): b64_bearer_token_creds = base64.b64encode(auth.encode('utf8')) params = {'grant_type': 'client_credentials'} headers = {} - headers['User-Agent'] = USER_AGENT + headers['User-Agent'] = self.user_agent headers['Authorization'] = 'Basic ' + b64_bearer_token_creds.decode('utf8') headers['Content-Type'] = 'application/x-www-form-urlencoded;charset=UTF-8' try: diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 31148ca..4f42448 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -17,6 +17,12 @@ import time +DEFAULT_USER_AGENT = 'python-TwitterAPI' +DEFAULT_CONNECTION_TIMEOUT = 5 +DEFAULT_STREAMING_TIMEOUT = 90 +DEFAULT_REST_TIMEOUT = 5 + + class TwitterAPI(object): """Access REST API or Streaming API resources. @@ -30,6 +36,12 @@ class TwitterAPI(object): or dict of URLs (ex. {'http':'http://SERVER', 'https':'https://SERVER'}) """ + # static properties to be overridden if desired + USER_AGENT = DEFAULT_USER_AGENT + CONNECTION_TIMEOUT = DEFAULT_CONNECTION_TIMEOUT + STREAMING_TIMEOUT = DEFAULT_STREAMING_TIMEOUT + REST_TIMEOUT = DEFAULT_REST_TIMEOUT + def __init__( self, consumer_key=None, @@ -59,7 +71,8 @@ def __init__( self.auth = OAuth2( consumer_key, consumer_secret, - proxies=self.proxies) + proxies=self.proxies, + user_agent=TwitterAPI.USER_AGENT) else: raise Exception('Unknown oAuth version') @@ -111,14 +124,14 @@ def request(self, resource, params=None, files=None, method_override=None): raise Exception('Endpoint "%s" unsupported' % endpoint) with requests.Session() as session: session.auth = self.auth - session.headers = {'User-Agent': USER_AGENT} + session.headers = {'User-Agent': TwitterAPI.USER_AGENT} method, subdomain = ENDPOINTS[endpoint] if method_override: method = method_override url = self._prepare_url(subdomain, resource) if 'stream' in subdomain: session.stream = True - timeout = STREAMING_TIMEOUT + timeout = TwitterAPI.STREAMING_TIMEOUT # always use 'delimited' for efficient stream parsing if not params: params = {} @@ -126,7 +139,7 @@ def request(self, resource, params=None, files=None, method_override=None): params['stall_warning'] = 'true' else: session.stream = False - timeout = REST_TIMEOUT + timeout = TwitterAPI.REST_TIMEOUT if method == 'POST': data = params params = None @@ -138,7 +151,7 @@ def request(self, resource, params=None, files=None, method_override=None): url, data=data, params=params, - timeout=(CONNECTION_TIMEOUT,timeout), + timeout=(TwitterAPI.CONNECTION_TIMEOUT,timeout), files=files, proxies=self.proxies) except (ConnectionError, ProtocolError, ReadTimeout, ReadTimeoutError, @@ -290,7 +303,7 @@ def _iter_stream(self): # check for stall (i.e. no data for 90 seconds) if not stall_timer: stall_timer = time.time() - elif time.time() - stall_timer > STREAMING_TIMEOUT: + elif time.time() - stall_timer > TwitterAPI.STREAMING_TIMEOUT: raise TwitterConnectionError('Twitter stream stalled') elif stall_timer: stall_timer = None diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 22f5acf..2a02c21 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -15,19 +15,12 @@ PROTOCOL = 'https' - DOMAIN = 'twitter.com' VERSION = '1.1' CURATOR_VERSION = 'broadcast/1' ADS_VERSION = '2' -USER_AGENT = 'python-TwitterAPI' - -CONNECTION_TIMEOUT = 5 -STREAMING_TIMEOUT = 90 -REST_TIMEOUT = 5 - ENDPOINTS = { # resource: (method, subdomain) From 7a54580f90674a8e8a304e3e60ee75294d6d26f0 Mon Sep 17 00:00:00 2001 From: Hammad Date: Sat, 5 Oct 2019 21:24:22 +0400 Subject: [PATCH 029/139] make timeouts overridable --- TwitterAPI/TwitterAPI.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 4f42448..f11cd89 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -15,12 +15,13 @@ import socket import ssl import time +import os -DEFAULT_USER_AGENT = 'python-TwitterAPI' -DEFAULT_CONNECTION_TIMEOUT = 5 -DEFAULT_STREAMING_TIMEOUT = 90 -DEFAULT_REST_TIMEOUT = 5 +DEFAULT_USER_AGENT = os.getenv('DEFAULT_USER_AGENT', 'python-TwitterAPI') +DEFAULT_CONNECTION_TIMEOUT = os.getenv('DEFAULT_CONNECTION_TIMEOUT', 5) +DEFAULT_STREAMING_TIMEOUT = os.getenv('DEFAULT_STREAMING_TIMEOUT', 90) +DEFAULT_REST_TIMEOUT = os.getenv('DEFAULT_REST_TIMEOUT', 5) class TwitterAPI(object): @@ -72,7 +73,7 @@ def __init__( consumer_key, consumer_secret, proxies=self.proxies, - user_agent=TwitterAPI.USER_AGENT) + user_agent=self.USER_AGENT) else: raise Exception('Unknown oAuth version') @@ -124,14 +125,14 @@ def request(self, resource, params=None, files=None, method_override=None): raise Exception('Endpoint "%s" unsupported' % endpoint) with requests.Session() as session: session.auth = self.auth - session.headers = {'User-Agent': TwitterAPI.USER_AGENT} + session.headers = {'User-Agent': self.USER_AGENT} method, subdomain = ENDPOINTS[endpoint] if method_override: method = method_override url = self._prepare_url(subdomain, resource) if 'stream' in subdomain: session.stream = True - timeout = TwitterAPI.STREAMING_TIMEOUT + timeout = self.STREAMING_TIMEOUT # always use 'delimited' for efficient stream parsing if not params: params = {} @@ -139,7 +140,7 @@ def request(self, resource, params=None, files=None, method_override=None): params['stall_warning'] = 'true' else: session.stream = False - timeout = TwitterAPI.REST_TIMEOUT + timeout = self.REST_TIMEOUT if method == 'POST': data = params params = None @@ -151,7 +152,7 @@ def request(self, resource, params=None, files=None, method_override=None): url, data=data, params=params, - timeout=(TwitterAPI.CONNECTION_TIMEOUT,timeout), + timeout=(self.CONNECTION_TIMEOUT, timeout), files=files, proxies=self.proxies) except (ConnectionError, ProtocolError, ReadTimeout, ReadTimeoutError, From dca74a6466fe1e936cac1873e2d84488d09bf198 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 28 Oct 2019 14:06:40 -0400 Subject: [PATCH 030/139] Merged HammadH-variable-timeouts --- CHANGES.txt | 2 ++ TwitterAPI/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 52c258e..3385b38 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -133,3 +133,5 @@ v2.5.8, 25 Dec 2018 -- Fixed bug in TwitterPager, repeating results. v2.5.9, 14 Jan 2019 -- Fixed bug in TwitterPager, stalling but not exiting when no more results. Moved USER_AGENT and TIMEOUT constants to TwitterAPI class static variables. + +v2.5.10, 28 Oct 2019 -- Merged HammadH-variable-timeouts diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 2840391..8ea7d4f 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.5.9' +__version__ = '2.5.10' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 110891a49e0ac760387e532ef7ff1442663cdb4f Mon Sep 17 00:00:00 2001 From: Rick Smith Date: Thu, 5 Mar 2020 16:54:19 +0000 Subject: [PATCH 031/139] Updated endpoints with new Labs functionality and now handling PUT method in request --- TwitterAPI/TwitterAPI.py | 31 ++++++++++++++++++++++--------- TwitterAPI/constants.py | 15 +++++++++++++-- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index f11cd89..2a281a7 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -90,6 +90,11 @@ def _prepare_url(self, subdomain, path): DOMAIN, ADS_VERSION, path) + elif subdomain == 'api' and 'labs/' in path: + return '%s://%s.%s/%s' % (PROTOCOL, + subdomain, + DOMAIN, + path) else: return '%s://%s.%s/%s/%s.json' % (PROTOCOL, subdomain, @@ -146,15 +151,23 @@ def request(self, resource, params=None, files=None, method_override=None): params = None else: data = None - try: - r = session.request( - method, - url, - data=data, - params=params, - timeout=(self.CONNECTION_TIMEOUT, timeout), - files=files, - proxies=self.proxies) + try: + if method == 'PUT': + session.headers['Content-type'] = 'application/json' + data = params + r = session.request( + method, + url, + json=data) + else: + r = session.request( + method, + url, + data=data, + params=params, + timeout=(self.CONNECTION_TIMEOUT, timeout), + files=files, + proxies=self.proxies) except (ConnectionError, ProtocolError, ReadTimeout, ReadTimeoutError, SSLError, ssl.SSLError, socket.error) as e: raise TwitterConnectionError(e) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 2a02c21..d75a8c0 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -198,7 +198,7 @@ 'stats/jobs/accounts/:PARAM/:PARAM': ('DELETE', 'ads-api'), # ACCOUNT ID, JOB ID 'stats/jobs/summaries': ('GET', 'ads-api'), - # NEW ACCOUNT ACTIVITY WEBHOOK API (BETA) + # ACCOUNT ACTIVITY WEBHOOK API 'account_activity/all/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME 'account_activity/all/:PARAM/subscriptions': ('DELETE', 'api'), # ENVIRONMENT NAME @@ -219,5 +219,16 @@ 'tweets/search/30day/:PARAM': ('GET', 'api'), # LABEL 'tweets/search/30day/:PARAM/counts': ('GET', 'api'), # LABEL 'tweets/search/fullarchive/:PARAM': ('GET', 'api'), # LABEL - 'tweets/search/fullarchive/:PARAM/counts': ('GET', 'api') # LABEL + 'tweets/search/fullarchive/:PARAM/counts': ('GET', 'api'), # LABEL + + #LABS API (BETAS) WILL NEED APPLICATION APPROVAL + 'labs/2/tweets/:PARAM': ('GET', 'api'), # TWEET ID + 'labs/2/tweets': ('GET', 'api'), + 'labs/2/tweets/search': ('GET', 'api'), + 'labs/2/tweets/:PARAM/hidden': ('PUT', 'api'), # TWEET ID + 'labs/2/users/:PARAM': ('GET', 'api'), # USER ID + 'labs/2/users': ('GET', 'api'), + + 'labs/1/tweets/metrics/private': ('GET', 'api') + } From 2c52db50046f85b9220d9c55c6b61342b62a4478 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 10 Apr 2020 15:28:19 -0400 Subject: [PATCH 032/139] Merged RickRedSix patch. Added Labs API. --- CHANGES.txt | 6 ++++-- TwitterAPI/TwitterAPI.py | 2 +- TwitterAPI/__init__.py | 2 +- TwitterAPI/constants.py | 5 ++--- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 3385b38..c254e10 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -127,11 +127,13 @@ v2.5.5, 07 Oct 2018 -- Merged x0139 patch. Added typing indicator endpoint. v2.5.6, 09 Oct 2018 -- Fixed TwitterPager. -v2.5.7, 15 Dec 2018 -- Merged HyperManTT path. Added subscripts/list endpoint. +v2.5.7, 15 Dec 2018 -- Merged HyperManTT patch. Added subscripts/list endpoint. v2.5.8, 25 Dec 2018 -- Fixed bug in TwitterPager, repeating results. v2.5.9, 14 Jan 2019 -- Fixed bug in TwitterPager, stalling but not exiting when no more results. Moved USER_AGENT and TIMEOUT constants to TwitterAPI class static variables. -v2.5.10, 28 Oct 2019 -- Merged HammadH-variable-timeouts +v2.5.10, 28 Oct 2019 -- Merged HammadH-variable-timeouts. + +v2.5.11, 28 Oct 2019 -- Merged RickRedSix patch. Added Labs API. diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 2a281a7..e356fda 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -91,7 +91,7 @@ def _prepare_url(self, subdomain, path): ADS_VERSION, path) elif subdomain == 'api' and 'labs/' in path: - return '%s://%s.%s/%s' % (PROTOCOL, + return '%s://%s.%s/%s' % (PROTOCOL, subdomain, DOMAIN, path) diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 8ea7d4f..215f253 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.5.10' +__version__ = '2.5.11' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index d75a8c0..717491e 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -219,7 +219,7 @@ 'tweets/search/30day/:PARAM': ('GET', 'api'), # LABEL 'tweets/search/30day/:PARAM/counts': ('GET', 'api'), # LABEL 'tweets/search/fullarchive/:PARAM': ('GET', 'api'), # LABEL - 'tweets/search/fullarchive/:PARAM/counts': ('GET', 'api'), # LABEL + 'tweets/search/fullarchive/:PARAM/counts': ('GET', 'api'), # LABEL #LABS API (BETAS) WILL NEED APPLICATION APPROVAL 'labs/2/tweets/:PARAM': ('GET', 'api'), # TWEET ID @@ -229,6 +229,5 @@ 'labs/2/users/:PARAM': ('GET', 'api'), # USER ID 'labs/2/users': ('GET', 'api'), - 'labs/1/tweets/metrics/private': ('GET', 'api') - + 'labs/1/tweets/metrics/private': ('GET', 'api') } From c64185fc85d3b98e56c6e041a20ddf47952db4e9 Mon Sep 17 00:00:00 2001 From: "Piotr S. Staszewski" Date: Mon, 13 Jul 2020 10:26:24 +0200 Subject: [PATCH 033/139] Make space for handling upstream-passed error message --- TwitterAPI/TwitterError.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/TwitterAPI/TwitterError.py b/TwitterAPI/TwitterError.py index 23010d8..249e281 100644 --- a/TwitterAPI/TwitterError.py +++ b/TwitterAPI/TwitterError.py @@ -25,14 +25,16 @@ class TwitterRequestError(TwitterError): """Raised when request fails""" - def __init__(self, status_code): - if status_code >= 500: - msg = 'Twitter internal error (you may re-try)' - else: - msg = 'Twitter request failed' + def __init__(self, status_code, msg=None): + if msg is None: + if status_code >= 500: + msg = 'Twitter internal error (you may re-try)' + else: + msg = 'Twitter request failed' logging.info('Status code %d: %s' % (status_code, msg)) super(Exception, self).__init__(msg) self.status_code = status_code - + self.msg = msg + def __str__(self): - return '%s (%d)' % (self.args[0], self.status_code) + return '%s (%d): %s' % (self.args[0], self.status_code, self.msg) From d933ac37ca9e18e7307e7912ed4b04bce19f3417 Mon Sep 17 00:00:00 2001 From: "Piotr S. Staszewski" Date: Mon, 13 Jul 2020 10:26:57 +0200 Subject: [PATCH 034/139] Use Twitter-provided text for error message As it may contain very useful information, e.g. bad params, malformed query etc. --- TwitterAPI/TwitterAPI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index e356fda..e3574ea 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -217,7 +217,7 @@ def get_iterator(self): :raises: TwitterConnectionError, TwitterRequestError """ if self.response.status_code != 200: - raise TwitterRequestError(self.response.status_code) + raise TwitterRequestError(self.response.status_code, msg=self.response.text) if self.stream: return iter(_StreamingIterable(self.response)) From 6b9d393d090e4ec47665bef25960987afb3c37a9 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 13 Jul 2020 10:07:18 -0400 Subject: [PATCH 035/139] drbig patch - improved error reporting --- CHANGES.txt | 4 +++- TwitterAPI/__init__.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index c254e10..cb04930 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -136,4 +136,6 @@ v2.5.9, 14 Jan 2019 -- Fixed bug in TwitterPager, stalling but not exiting wh v2.5.10, 28 Oct 2019 -- Merged HammadH-variable-timeouts. -v2.5.11, 28 Oct 2019 -- Merged RickRedSix patch. Added Labs API. +v2.5.11, 10 Apr 2020 -- Merged RickRedSix patch. Added Labs API. + +v2.5.12, 13 Jul 2020 -- Merged drbig patch. Improved error reporting. diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 215f253..fe881ba 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.5.11' +__version__ = '2.5.12' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 144781dfb86c51ff75bee4a9274bb099d48e2fb7 Mon Sep 17 00:00:00 2001 From: "Piotr S. Staszewski" Date: Tue, 14 Jul 2020 11:21:26 +0200 Subject: [PATCH 036/139] Make exception raising configurable on TwitterPager For Free APIs continuing on all network errors etc. is probably fine, but very much not so on the paid APIs (they do count your request even if you got read timeout... speaking from experience). --- TwitterAPI/TwitterPager.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py index 4787199..2778296 100644 --- a/TwitterAPI/TwitterPager.py +++ b/TwitterAPI/TwitterPager.py @@ -20,6 +20,10 @@ class TwitterPager(object): :param params: Dictionary of resource parameters """ + # static properties to be overridden if desired + RAISE_CONNECTION_ERROR = False + RAISE_REQUEST_ERROR = False + def __init__(self, api, resource, params=None): self.api = api self.resource = resource @@ -106,8 +110,14 @@ def get_iterator(self, wait=5, new_tweets=False): continue except TwitterRequestError as e: + if self.RAISE_REQUEST_ERROR: + raise + if e.status_code < 500: raise continue except TwitterConnectionError: - continue \ No newline at end of file + if self.RAISE_CONNECTION_ERROR: + raise + + continue From eab4b249eaeea167b323f9b5b533989565c41084 Mon Sep 17 00:00:00 2001 From: Kevin Reuning Date: Tue, 14 Jul 2020 16:18:30 -0400 Subject: [PATCH 037/139] Update constants.py Added media/metadata/create endpoint. --- TwitterAPI/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 717491e..cfb7054 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -110,6 +110,7 @@ 'lists/update': ('POST', 'api'), 'media/upload': ('POST', 'upload'), + 'media/metadata/create': ('POST', 'upload'), 'mutes/users/create': ('POST', 'api'), 'mutes/users/destroy': ('POST', 'api'), From be02d8794c5eb2611ffe311cd349d4ec5cb89192 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 17 Jul 2020 10:16:08 -0400 Subject: [PATCH 038/139] Merged drbig patch for error control --- CHANGES.txt | 2 ++ TwitterAPI/TwitterPager.py | 3 +-- TwitterAPI/__init__.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index cb04930..cfe2dce 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -139,3 +139,5 @@ v2.5.10, 28 Oct 2019 -- Merged HammadH-variable-timeouts. v2.5.11, 10 Apr 2020 -- Merged RickRedSix patch. Added Labs API. v2.5.12, 13 Jul 2020 -- Merged drbig patch. Improved error reporting. + +v2.5.13, 17 Jul 2020 -- Merged drbig patch. Override exception handling in TwitterPager. diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py index 2778296..9a06b57 100644 --- a/TwitterAPI/TwitterPager.py +++ b/TwitterAPI/TwitterPager.py @@ -112,12 +112,11 @@ def get_iterator(self, wait=5, new_tweets=False): except TwitterRequestError as e: if self.RAISE_REQUEST_ERROR: raise - if e.status_code < 500: raise continue + except TwitterConnectionError: if self.RAISE_CONNECTION_ERROR: raise - continue diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index fe881ba..b8b7bfd 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.5.12' +__version__ = '2.5.13' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From e9d17435298c76ea7b002a52f81093b3eb540174 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 17 Jul 2020 10:23:22 -0400 Subject: [PATCH 039/139] Merged reuning patch to add new endpoint --- CHANGES.txt | 1 + TwitterAPI/constants.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index cfe2dce..fa7d6bd 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -141,3 +141,4 @@ v2.5.11, 10 Apr 2020 -- Merged RickRedSix patch. Added Labs API. v2.5.12, 13 Jul 2020 -- Merged drbig patch. Improved error reporting. v2.5.13, 17 Jul 2020 -- Merged drbig patch. Override exception handling in TwitterPager. + Merged reuning patch. Added new endpoint. diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index cfb7054..cef6d3c 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -109,8 +109,8 @@ 'lists/subscriptions': ('GET', 'api'), 'lists/update': ('POST', 'api'), - 'media/upload': ('POST', 'upload'), 'media/metadata/create': ('POST', 'upload'), + 'media/upload': ('POST', 'upload'), 'mutes/users/create': ('POST', 'api'), 'mutes/users/destroy': ('POST', 'api'), From 5ede20d0265ffad0c872f5ab18082150dc394607 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Thu, 17 Sep 2020 11:44:44 -0400 Subject: [PATCH 040/139] Fix TwitterPager to iterate integer ids --- CHANGES.txt | 2 ++ TwitterAPI/TwitterPager.py | 26 +++++++++----------------- TwitterAPI/__init__.py | 2 +- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index fa7d6bd..fcd84e4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -142,3 +142,5 @@ v2.5.12, 13 Jul 2020 -- Merged drbig patch. Improved error reporting. v2.5.13, 17 Jul 2020 -- Merged drbig patch. Override exception handling in TwitterPager. Merged reuning patch. Added new endpoint. + +v2.5.14, 17 Sep 2020 -- Fix TwitterPager to iterate integer ids. diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py index 9a06b57..d19f4a5 100644 --- a/TwitterAPI/TwitterPager.py +++ b/TwitterAPI/TwitterPager.py @@ -20,10 +20,6 @@ class TwitterPager(object): :param params: Dictionary of resource parameters """ - # static properties to be overridden if desired - RAISE_CONNECTION_ERROR = False - RAISE_REQUEST_ERROR = False - def __init__(self, api, resource, params=None): self.api = api self.resource = resource @@ -57,12 +53,13 @@ def get_iterator(self, wait=5, new_tweets=False): id = None for item in it: item_count = item_count + 1 - if 'id' in item: - id = item['id'] - if 'code' in item: - if item['code'] in [130, 131]: - # Twitter service error - raise TwitterConnectionError(item) + if type(item) is dict: + if 'id' in item: + id = item['id'] + if 'code' in item: + if item['code'] in [130, 131]: + # Twitter service error + raise TwitterConnectionError(item) yield item # if a cursor is present, use it to get next page @@ -81,7 +78,7 @@ def get_iterator(self, wait=5, new_tweets=False): cursor = json['next'] cursor_param = 'next' - is_premium_search = 'query' in self.params + is_premium_search = self.params and 'query' in self.params # bail when no more results if cursor == 0: @@ -110,13 +107,8 @@ def get_iterator(self, wait=5, new_tweets=False): continue except TwitterRequestError as e: - if self.RAISE_REQUEST_ERROR: - raise if e.status_code < 500: raise continue - except TwitterConnectionError: - if self.RAISE_CONNECTION_ERROR: - raise - continue + continue \ No newline at end of file diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index b8b7bfd..7416161 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.5.13' +__version__ = '2.5.14' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 722f687beb3adf32c83c49d7bb81aaa5abadb0c3 Mon Sep 17 00:00:00 2001 From: Dylan Caponi Date: Fri, 2 Oct 2020 01:50:06 -0700 Subject: [PATCH 041/139] Add v2 endpoints --- TwitterAPI/constants.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index cef6d3c..f055e64 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -215,6 +215,20 @@ 'account_activity/webhooks/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME 'account_activity/webhooks/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME + # API V2 EARLY ACCESS + '2/tweets/:PARAM': ('GET', 'api'), # ID + '2/tweets?ids=:PARAM': ('GET', 'api'), # IDS + '2/users/:PARAM': ('GET', 'api'), # ID + '2/users?ids=:PARAM': ('GET', 'api'), # IDS + '2/users/by/username/:PARAM': ('GET', 'api'), # USERNAME + '2/users/by?usernames=:PARAM': ('GET', 'api'), # USERNAMES + '2/tweets/search/recent?query=:PARAM': ('GET', 'api'), # QUERY + '2/tweets/search/stream/rules': ('POST', 'api'), + '2/tweets/search/stream/rules': ('GET', 'api'), + '2/tweets/sample/stream': ('GET', 'api'), + '2/tweets/:id/hidden': ('PUT', 'api'), + + # PREMIUM SEARCH API 'tweets/search/30day/:PARAM': ('GET', 'api'), # LABEL @@ -223,6 +237,7 @@ 'tweets/search/fullarchive/:PARAM/counts': ('GET', 'api'), # LABEL #LABS API (BETAS) WILL NEED APPLICATION APPROVAL + 'labs/2/tweets/:PARAM': ('GET', 'api'), # TWEET ID 'labs/2/tweets': ('GET', 'api'), 'labs/2/tweets/search': ('GET', 'api'), From 6d857e524ac3953f2470cd3935d90d23e9c2f89e Mon Sep 17 00:00:00 2001 From: Dylan Caponi Date: Fri, 2 Oct 2020 13:22:01 -0700 Subject: [PATCH 042/139] Add version check --- TwitterAPI/TwitterAPI.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index e3574ea..1626521 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -78,6 +78,9 @@ def __init__( raise Exception('Unknown oAuth version') def _prepare_url(self, subdomain, path): + if self.version: + version = self.version + if subdomain == 'curator': return '%s://%s.%s/%s/%s.json' % (PROTOCOL, subdomain, @@ -90,6 +93,13 @@ def _prepare_url(self, subdomain, path): DOMAIN, ADS_VERSION, path) + elif version == '2': + return '%s://%s.%s/%s/%s' % (PROTOCOL, + subdomain, + DOMAIN, + version, + path) + elif subdomain == 'api' and 'labs/' in path: return '%s://%s.%s/%s' % (PROTOCOL, subdomain, @@ -114,7 +124,7 @@ def _get_endpoint(self, resource): else: return (resource, resource) - def request(self, resource, params=None, files=None, method_override=None): + def request(self, resource, params=None, files=None, method_override=None, version=None): """Request a Twitter REST API or Streaming API resource. :param resource: A valid Twitter endpoint (ex. "search/tweets") @@ -125,6 +135,7 @@ def request(self, resource, params=None, files=None, method_override=None): :returns: TwitterResponse :raises: TwitterConnectionError """ + self.version = version resource, endpoint = self._get_endpoint(resource) if endpoint not in ENDPOINTS: raise Exception('Endpoint "%s" unsupported' % endpoint) From 2396ab81f5d471e6db23e782e3cf62d83bc7118d Mon Sep 17 00:00:00 2001 From: Dylan Caponi Date: Fri, 2 Oct 2020 13:22:12 -0700 Subject: [PATCH 043/139] Add v2 endpoints --- TwitterAPI/constants.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index f055e64..87769b4 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -17,7 +17,7 @@ PROTOCOL = 'https' DOMAIN = 'twitter.com' -VERSION = '1.1' +# VERSION = '1.1' CURATOR_VERSION = 'broadcast/1' ADS_VERSION = '2' @@ -216,17 +216,18 @@ 'account_activity/webhooks/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME # API V2 EARLY ACCESS - '2/tweets/:PARAM': ('GET', 'api'), # ID - '2/tweets?ids=:PARAM': ('GET', 'api'), # IDS - '2/users/:PARAM': ('GET', 'api'), # ID - '2/users?ids=:PARAM': ('GET', 'api'), # IDS - '2/users/by/username/:PARAM': ('GET', 'api'), # USERNAME - '2/users/by?usernames=:PARAM': ('GET', 'api'), # USERNAMES - '2/tweets/search/recent?query=:PARAM': ('GET', 'api'), # QUERY - '2/tweets/search/stream/rules': ('POST', 'api'), - '2/tweets/search/stream/rules': ('GET', 'api'), - '2/tweets/sample/stream': ('GET', 'api'), - '2/tweets/:id/hidden': ('PUT', 'api'), + + 'tweets/:PARAM': ('GET', 'api'), # ID + 'tweets?ids=:PARAM': ('GET', 'api'), # IDS + 'users/:PARAM': ('GET', 'api'), # ID + 'users?ids=:PARAM': ('GET', 'api'), # IDS + 'users/by/username/:PARAM': ('GET', 'api'), # USERNAME + 'users/by?usernames=:PARAM': ('GET', 'api'), # USERNAMES + 'tweets/search/recent?query=:PARAM': ('GET', 'api'), # QUERY + 'tweets/search/stream/rules': ('POST', 'api'), + 'tweets/search/stream/rules': ('GET', 'api'), + 'tweets/sample/stream': ('GET', 'api'), + 'tweets/:id/hidden': ('PUT', 'api'), # PREMIUM SEARCH API From fc7427877dd949de274d787800bcad71b0899950 Mon Sep 17 00:00:00 2001 From: Dylan Caponi Date: Fri, 2 Oct 2020 13:24:56 -0700 Subject: [PATCH 044/139] Add user lookup v2 example --- examples/lookup_user_v2.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 examples/lookup_user_v2.py diff --git a/examples/lookup_user_v2.py b/examples/lookup_user_v2.py new file mode 100644 index 0000000..8b337ef --- /dev/null +++ b/examples/lookup_user_v2.py @@ -0,0 +1,16 @@ +# Documentation +# https://documenter.getpostman.com/view/9956214/T1LMiT5U#auth-info-784efcda-ed4c-4491-a4c0-a26470a67400 + +# You must opt-in to v2 here: +# https://developer.twitter.com/en/portal/opt-in + +from TwitterAPI import TwitterAPI, TwitterPager + +SCREEN_NAME = 'TheTweetOfGod' + +api = TwitterAPI(, + , + auth_type='oAuth2') + +r = api.request(f'users/by/username/:{SCREEN_NAME}', version='2') +print(r.json()['data']['id'] if r.status_code == 200 else 'PROBLEM: ' + r.text) From 890f9aa63c52ce74d3b04b8ecdcb40ea34eade2a Mon Sep 17 00:00:00 2001 From: Dylan Caponi Date: Fri, 2 Oct 2020 13:30:39 -0700 Subject: [PATCH 045/139] Uncomment VERSION --- TwitterAPI/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 87769b4..b0317df 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -17,7 +17,7 @@ PROTOCOL = 'https' DOMAIN = 'twitter.com' -# VERSION = '1.1' +VERSION = '1.1' CURATOR_VERSION = 'broadcast/1' ADS_VERSION = '2' From 16f53602fa7226901f31512988e90ce94459f006 Mon Sep 17 00:00:00 2001 From: Dylan Caponi Date: Fri, 2 Oct 2020 14:39:17 -0700 Subject: [PATCH 046/139] Add lookup_tweet_v2 example --- examples/lookup_tweet_v2.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 examples/lookup_tweet_v2.py diff --git a/examples/lookup_tweet_v2.py b/examples/lookup_tweet_v2.py new file mode 100644 index 0000000..e3ccd90 --- /dev/null +++ b/examples/lookup_tweet_v2.py @@ -0,0 +1,15 @@ +# You must opt-in to v2 here: +# https://developer.twitter.com/en/portal/opt-in + +from TwitterAPI import TwitterAPI + +TWEET_ID = '964575983633252353' + +api = TwitterAPI(, + , + auth_type='oAuth2') + +r = api.request(f'tweets/:{TWEET_ID}' , version='2') +tweet = r.json() +print(f"{tweet['data']['id']}: {tweet['data']['text']}" if r.status_code == 200 + else f"PROBLEM: {r.text}") From a39921a1d7c8f882b2debee2ba93f8049606205d Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 19 Oct 2020 17:25:19 -0400 Subject: [PATCH 047/139] Supports both Version 1.1 and Version 2 (early release) --- CHANGES.txt | 4 ++ README.rst | 10 ++++- TwitterAPI/TwitterAPI.py | 75 ++++++++++++++++++++----------- TwitterAPI/TwitterError.py | 21 +++++++-- TwitterAPI/TwitterPager.py | 90 ++++++++++++++++++++----------------- TwitterAPI/__init__.py | 2 +- TwitterAPI/constants.py | 32 +++++++------ examples/v2_PageTweets.py | 19 ++++++++ examples/v2_SearchTweets.py | 22 +++++++++ examples/v2_StreamTweets.py | 36 +++++++++++++++ 10 files changed, 224 insertions(+), 87 deletions(-) create mode 100644 examples/v2_PageTweets.py create mode 100644 examples/v2_SearchTweets.py create mode 100644 examples/v2_StreamTweets.py diff --git a/CHANGES.txt b/CHANGES.txt index fcd84e4..1223303 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -144,3 +144,7 @@ v2.5.13, 17 Jul 2020 -- Merged drbig patch. Override exception handling in Twi Merged reuning patch. Added new endpoint. v2.5.14, 17 Sep 2020 -- Fix TwitterPager to iterate integer ids. + +v2.6.0, 04 Oct 2020 -- Merged dylancaponi-twitter-v2 with V2 support. + Made several modifications to complete V2 support. + Dropping Python 2.7 compatibility. \ No newline at end of file diff --git a/README.rst b/README.rst index ba1c8eb..5f835bc 100755 --- a/README.rst +++ b/README.rst @@ -10,10 +10,18 @@ :alt: Join the chat at https://gitter.im/geduldig/TwitterAPI :target: https://gitter.im/geduldig/TwitterAPI?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge -TwitterAPI is a Python package for accessing Twitter's REST APIs and Streaming APIs. It supports OAuth 1.0 and OAuth 2.0 authentication. And, it works with the latest Python versions in both 2.x and 3.x branches. +TwitterAPI is a Python package for accessing Twitter's REST APIs and Streaming APIs. It supports both Version 1.1 and Version 2 endpoints. REST APIs that are supported are: Public API, Collections API, Curator API, Ads API, Webhook API, Premium Search API. +NEW -- Twitter API Version 2!! +------------------------------ +To access Version 2 endpoints, specify the API version:: + + api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret, api_version='2') + +Check the examples folder for more information on making requests using Version 2 endpoints. + Installation ------------ From the command line:: diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 1626521..44bb2a7 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -50,14 +50,22 @@ def __init__( access_token_key=None, access_token_secret=None, auth_type='oAuth1', - proxy_url=None): + proxy_url=None, + api_version=VERSION): """Initialize with your Twitter application credentials""" + # if there are multiple API versions, this will be the default version which can + # also be overridden by specifying the version when calling the request method. + self.version = api_version + + # Optional proxy or proxies. if isinstance(proxy_url, dict): self.proxies = proxy_url elif proxy_url is not None: self.proxies = {'https': proxy_url} else: self.proxies = None + + # Twitter supports two types of authentication. if auth_type == 'oAuth1': if not all([consumer_key, consumer_secret, access_token_key, access_token_secret]): raise Exception('Missing authentication parameter') @@ -77,10 +85,7 @@ def __init__( else: raise Exception('Unknown oAuth version') - def _prepare_url(self, subdomain, path): - if self.version: - version = self.version - + def _prepare_url(self, subdomain, path, api_version): if subdomain == 'curator': return '%s://%s.%s/%s/%s.json' % (PROTOCOL, subdomain, @@ -93,24 +98,25 @@ def _prepare_url(self, subdomain, path): DOMAIN, ADS_VERSION, path) - elif version == '2': - return '%s://%s.%s/%s/%s' % (PROTOCOL, - subdomain, - DOMAIN, - version, - path) - elif subdomain == 'api' and 'labs/' in path: return '%s://%s.%s/%s' % (PROTOCOL, subdomain, DOMAIN, path) - else: + elif api_version == '1.1': return '%s://%s.%s/%s/%s.json' % (PROTOCOL, subdomain, DOMAIN, - VERSION, + api_version, path) + elif api_version == '2': + return '%s://%s.%s/%s/%s' % (PROTOCOL, + subdomain, + DOMAIN, + api_version, + path) + else: + raise Exception('Unsupported API version') def _get_endpoint(self, resource): """Substitute any parameters in the resource path with :PARAM.""" @@ -124,18 +130,18 @@ def _get_endpoint(self, resource): else: return (resource, resource) - def request(self, resource, params=None, files=None, method_override=None, version=None): + def request(self, resource, params=None, files=None, method_override=None, api_version=None): """Request a Twitter REST API or Streaming API resource. :param resource: A valid Twitter endpoint (ex. "search/tweets") :param params: Dictionary with endpoint parameters or None (default) :param files: Dictionary with multipart-encoded file or None (default) :param method_override: Request method to override or None (default) + :param api_version: override default API version or None (default) :returns: TwitterResponse :raises: TwitterConnectionError """ - self.version = version resource, endpoint = self._get_endpoint(resource) if endpoint not in ENDPOINTS: raise Exception('Endpoint "%s" unsupported' % endpoint) @@ -145,8 +151,10 @@ def request(self, resource, params=None, files=None, method_override=None, versi method, subdomain = ENDPOINTS[endpoint] if method_override: method = method_override - url = self._prepare_url(subdomain, resource) - if 'stream' in subdomain: + if not api_version: + api_version = self.version + url = self._prepare_url(subdomain, resource, api_version) + if api_version == '1.1' and 'stream' in subdomain: session.stream = True timeout = self.STREAMING_TIMEOUT # always use 'delimited' for efficient stream parsing @@ -154,16 +162,29 @@ def request(self, resource, params=None, files=None, method_override=None, versi params = {} params['delimited'] = 'length' params['stall_warning'] = 'true' + elif api_version == '2' and resource.endswith('/stream'): + session.stream = True + timeout = self.STREAMING_TIMEOUT else: session.stream = False timeout = self.REST_TIMEOUT + # if method == 'POST': + # data = params + # params = None + # else: + # data = None + d = p = j = None if method == 'POST': - data = params - params = None + if api_version == '1.1': + d = params + else: + j = params + elif method == 'PUT': + j = params else: - data = None - try: - if method == 'PUT': + p = params + try: + if False and method == 'PUT': session.headers['Content-type'] = 'application/json' data = params r = session.request( @@ -174,8 +195,12 @@ def request(self, resource, params=None, files=None, method_override=None, versi r = session.request( method, url, - data=data, - params=params, + # data=data, + # params=params, + # json=params, + data=d, + params=p, + json=j, timeout=(self.CONNECTION_TIMEOUT, timeout), files=files, proxies=self.proxies) diff --git a/TwitterAPI/TwitterError.py b/TwitterAPI/TwitterError.py index 249e281..0ebfa21 100644 --- a/TwitterAPI/TwitterError.py +++ b/TwitterAPI/TwitterError.py @@ -4,6 +4,7 @@ import logging +import json class TwitterError(Exception): @@ -17,7 +18,7 @@ class TwitterConnectionError(TwitterError): """Raised when the connection needs to be re-established""" def __init__(self, value): - super(Exception, self).__init__(value) + super().__init__(value) logging.warning('%s %s' % (type(value), value)) @@ -32,9 +33,23 @@ def __init__(self, status_code, msg=None): else: msg = 'Twitter request failed' logging.info('Status code %d: %s' % (status_code, msg)) - super(Exception, self).__init__(msg) + super().__init__(msg) self.status_code = status_code self.msg = msg def __str__(self): - return '%s (%d): %s' % (self.args[0], self.status_code, self.msg) + return '%s (%d): %s' % (self.args, self.status_code, self.msg) + + + def __iter__(self): + try: + msg = json.loads(self.msg) + if 'errors' in msg: + for error in msg['errors']: + yield error['message'] + elif 'detail' in msg: + yield msg['detail'] + else: + yield self.msg + except: + yield self.msg \ No newline at end of file diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py index d19f4a5..554d0c9 100644 --- a/TwitterAPI/TwitterPager.py +++ b/TwitterAPI/TwitterPager.py @@ -34,25 +34,25 @@ def get_iterator(self, wait=5, new_tweets=False): :param new_tweets: Boolean determining the search direction. False (default) retrieves old results. True retrieves current results. + :param version: override default API version or None (default) :returns: JSON objects containing statuses, errors or other return info. - :raises: TwitterRequestError """ elapsed = 0 while True: try: - # get one page of results + # REQUEST ONE PAGE OF RESULTS... start = time.time() r = self.api.request(self.resource, self.params) it = r.get_iterator() if new_tweets: it = reversed(list(it)) - # yield each item in the page + # YIELD FOR EACH ITEM IN THE PAGE... item_count = 0 id = None for item in it: - item_count = item_count + 1 + item_count += 1 if type(item) is dict: if 'id' in item: id = item['id'] @@ -62,49 +62,59 @@ def get_iterator(self, wait=5, new_tweets=False): raise TwitterConnectionError(item) yield item - # if a cursor is present, use it to get next page - # otherwise, use id to get next page json = r.json() - cursor = -1 - if new_tweets and 'previous_cursor' in json: - cursor = json['previous_cursor'] - cursor_param = 'cursor' - elif not new_tweets and 'next_cursor' in json: - cursor = json['next_cursor'] - cursor_param = 'cursor' - elif not new_tweets and 'next' in json: - # 'next' is used by Premium Search, so only - # works searching back in time - cursor = json['next'] - cursor_param = 'next' - - is_premium_search = self.params and 'query' in self.params - - # bail when no more results - if cursor == 0: - break - elif cursor == -1 and is_premium_search: - break - elif not new_tweets and item_count == 0: - break - - # sleep before getting another page of results + + # CHECK FOR NEXT PAGE OR BAIL... + if self.api.version == '1.1': + # if a cursor is present, use it to get next page + # otherwise, use id to get next page + is_premium_search = self.params and 'query' in self.params and self.api.version == '1.1' + cursor = -1 + if new_tweets and 'previous_cursor' in json: + cursor = json['previous_cursor'] + cursor_param = 'cursor' + elif not new_tweets: + if 'next_cursor' in json: + cursor = json['next_cursor'] + cursor_param = 'cursor' + elif 'next' in json: + # 'next' is used by Premium Search (OLD searches only) + cursor = json['next'] + cursor_param = 'next' + + # bail when no more results + if cursor == 0: + break + elif cursor == -1 and is_premium_search: + break + elif not new_tweets and item_count == 0: + break + else: # VERSION 2 + meta = json['meta'] + + # SLEEP... elapsed = time.time() - start pause = wait - elapsed if elapsed < wait else 0 time.sleep(pause) - # waiting for new results to come in... - # get a page with cursor if present, or with id if not - # a Premium search (i.e. 'query' is not a parameter) - if cursor != -1: - self.params[cursor_param] = cursor - elif id is not None and not is_premium_search: + # SETUP REQUEST FOR NEXT PAGE... + if self.api.version == '1.1': + # get a page with cursor if present, or with id if not + # a Premium search (i.e. 'query' is not a parameter) + if cursor != -1: + self.params[cursor_param] = cursor + elif id is not None and not is_premium_search: + if new_tweets: + self.params['since_id'] = str(id) + else: + self.params['max_id'] = str(id - 1) + else: + continue + else: # VERSION 2 if new_tweets: - self.params['since_id'] = str(id) + self.params['since_id'] = meta['newest_id'] else: - self.params['max_id'] = str(id - 1) - else: - continue + self.params['next_token'] = meta['next_token'] except TwitterRequestError as e: if e.status_code < 500: diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 7416161..7646013 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.5.14' +__version__ = '2.6.0' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index b0317df..f897cf6 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -214,21 +214,6 @@ 'account_activity/webhooks/:PARAM': ('DELETE', 'api'), # WEBHOOK ID 'account_activity/webhooks/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME 'account_activity/webhooks/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME - - # API V2 EARLY ACCESS - - 'tweets/:PARAM': ('GET', 'api'), # ID - 'tweets?ids=:PARAM': ('GET', 'api'), # IDS - 'users/:PARAM': ('GET', 'api'), # ID - 'users?ids=:PARAM': ('GET', 'api'), # IDS - 'users/by/username/:PARAM': ('GET', 'api'), # USERNAME - 'users/by?usernames=:PARAM': ('GET', 'api'), # USERNAMES - 'tweets/search/recent?query=:PARAM': ('GET', 'api'), # QUERY - 'tweets/search/stream/rules': ('POST', 'api'), - 'tweets/search/stream/rules': ('GET', 'api'), - 'tweets/sample/stream': ('GET', 'api'), - 'tweets/:id/hidden': ('PUT', 'api'), - # PREMIUM SEARCH API @@ -237,8 +222,9 @@ 'tweets/search/fullarchive/:PARAM': ('GET', 'api'), # LABEL 'tweets/search/fullarchive/:PARAM/counts': ('GET', 'api'), # LABEL - #LABS API (BETAS) WILL NEED APPLICATION APPROVAL + # LABS API (BETAS) WILL NEED APPLICATION APPROVAL + 'labs/1/tweets/metrics/private': ('GET', 'api'), 'labs/2/tweets/:PARAM': ('GET', 'api'), # TWEET ID 'labs/2/tweets': ('GET', 'api'), 'labs/2/tweets/search': ('GET', 'api'), @@ -246,5 +232,17 @@ 'labs/2/users/:PARAM': ('GET', 'api'), # USER ID 'labs/2/users': ('GET', 'api'), - 'labs/1/tweets/metrics/private': ('GET', 'api') + # API V2 EARLY ACCESS + + 'tweets': ('GET', 'api'), + 'tweets/:PARAM/hidden': ('PUT', 'api'), # ID + 'tweets/:PARAM': ('GET', 'api'), # ID + 'tweets/sample/stream': ('GET', 'api'), + 'tweets/search/recent': ('GET', 'api'), + 'tweets/search/stream': ('GET', 'api'), + 'tweets/search/stream/rules': ('POST', 'api'), # use method_override for 'GET' + 'users': ('GET', 'api'), + 'users/:PARAM': ('GET', 'api'), # ID + 'users/by': ('GET', 'api'), + 'users/by/username/:PARAM': ('GET', 'api'), # USERNAME } diff --git a/examples/v2_PageTweets.py b/examples/v2_PageTweets.py new file mode 100644 index 0000000..de7b36c --- /dev/null +++ b/examples/v2_PageTweets.py @@ -0,0 +1,19 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2'') + pager = TwitterPager(api, 'tweets/search/recent', {'query':'pizza'}) + for item in pager.get_iterator(new_tweets=False): + print(item) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2_SearchTweets.py b/examples/v2_SearchTweets.py new file mode 100644 index 0000000..5b9e4a8 --- /dev/null +++ b/examples/v2_SearchTweets.py @@ -0,0 +1,22 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request('tweets/search/recent', {'query':'pizza', 'tweet.fields':'public_metrics'}) + + for item in r: + print(item) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2_StreamTweets.py b/examples/v2_StreamTweets.py new file mode 100644 index 0000000..d01cfac --- /dev/null +++ b/examples/v2_StreamTweets.py @@ -0,0 +1,36 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + + # ADD STREAM RULES + + r = api.request('tweets/search/stream/rules', {'add': [{'value':'pizza'}]}) + print(f'[{r.status_code}] RULE ADDED: {r.text}') + if r.status_code != 201: exit() + + # GET STREAM RULES + + r = api.request('tweets/search/stream/rules', method_override='GET') + print(f'[{r.status_code}] RULES: {r.text}') + if r.status_code != 200: exit() + + # START STREAM + + r = api.request('tweets/search/stream') + print(f'[{r.status_code}] START...') + if r.status_code != 200: exit() + for item in r: + print(item) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file From 54a341769edeed85785857b352c3e44d15c55fa9 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Tue, 20 Oct 2020 14:49:34 -0400 Subject: [PATCH 048/139] updated examples --- README.rst | 2 +- TwitterAPI/__init__.py | 2 +- examples/lookup_tweet_v2.py | 15 ------------ examples/lookup_user_v2.py | 16 ------------- examples/{ => v1.1}/connect_using_login.py | 0 examples/{ => v1.1}/connect_using_oauth.py | 0 examples/{ => v1.1}/delete_last_tweet.py | 0 examples/{ => v1.1}/direct_message.py | 0 examples/{ => v1.1}/dump_timeline.py | 0 examples/{ => v1.1}/logging.py | 0 examples/{ => v1.1}/lookup_tweet.py | 0 examples/{ => v1.1}/lookup_user.py | 0 examples/{ => v1.1}/page_tweets.py | 0 examples/{ => v1.1}/post_tweet.py | 0 examples/{ => v1.1}/premium_search.py | 0 examples/{ => v1.1}/sample_frequency.py | 0 examples/{ => v1.1}/search_tweets.py | 0 examples/{ => v1.1}/stream_tweets.py | 0 examples/{ => v1.1}/unfollow_friends.py | 0 examples/{ => v1.1}/upload_image.py | 0 examples/{ => v1.1}/upload_tweet.py | 0 examples/{ => v1.1}/upload_video.py | 0 .../lookup_tweet_v2.py} | 4 +++- examples/v2/lookup_user_v2.py | 24 +++++++++++++++++++ .../{v2_PageTweets.py => v2/page_tweets.py} | 4 +++- examples/v2/search_tweets.py | 24 +++++++++++++++++++ .../stream_tweets.py} | 4 +++- 27 files changed, 59 insertions(+), 36 deletions(-) delete mode 100644 examples/lookup_tweet_v2.py delete mode 100644 examples/lookup_user_v2.py rename examples/{ => v1.1}/connect_using_login.py (100%) rename examples/{ => v1.1}/connect_using_oauth.py (100%) rename examples/{ => v1.1}/delete_last_tweet.py (100%) rename examples/{ => v1.1}/direct_message.py (100%) rename examples/{ => v1.1}/dump_timeline.py (100%) rename examples/{ => v1.1}/logging.py (100%) rename examples/{ => v1.1}/lookup_tweet.py (100%) rename examples/{ => v1.1}/lookup_user.py (100%) rename examples/{ => v1.1}/page_tweets.py (100%) rename examples/{ => v1.1}/post_tweet.py (100%) rename examples/{ => v1.1}/premium_search.py (100%) rename examples/{ => v1.1}/sample_frequency.py (100%) rename examples/{ => v1.1}/search_tweets.py (100%) rename examples/{ => v1.1}/stream_tweets.py (100%) rename examples/{ => v1.1}/unfollow_friends.py (100%) rename examples/{ => v1.1}/upload_image.py (100%) rename examples/{ => v1.1}/upload_tweet.py (100%) rename examples/{ => v1.1}/upload_video.py (100%) rename examples/{v2_SearchTweets.py => v2/lookup_tweet_v2.py} (83%) create mode 100644 examples/v2/lookup_user_v2.py rename examples/{v2_PageTweets.py => v2/page_tweets.py} (85%) create mode 100644 examples/v2/search_tweets.py rename examples/{v2_StreamTweets.py => v2/stream_tweets.py} (96%) diff --git a/README.rst b/README.rst index 5f835bc..9932a36 100755 --- a/README.rst +++ b/README.rst @@ -16,7 +16,7 @@ REST APIs that are supported are: Public API, Collections API, Curator API, Ads NEW -- Twitter API Version 2!! ------------------------------ -To access Version 2 endpoints, specify the API version:: +By default TwitterAPI will permit only Version 1.1 endpoints. To start using Verson 2 endpoint you must supply the api_version parameter: api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret, api_version='2') diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 7646013..457279b 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.0' +__version__ = '2.6.0.1' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/examples/lookup_tweet_v2.py b/examples/lookup_tweet_v2.py deleted file mode 100644 index e3ccd90..0000000 --- a/examples/lookup_tweet_v2.py +++ /dev/null @@ -1,15 +0,0 @@ -# You must opt-in to v2 here: -# https://developer.twitter.com/en/portal/opt-in - -from TwitterAPI import TwitterAPI - -TWEET_ID = '964575983633252353' - -api = TwitterAPI(, - , - auth_type='oAuth2') - -r = api.request(f'tweets/:{TWEET_ID}' , version='2') -tweet = r.json() -print(f"{tweet['data']['id']}: {tweet['data']['text']}" if r.status_code == 200 - else f"PROBLEM: {r.text}") diff --git a/examples/lookup_user_v2.py b/examples/lookup_user_v2.py deleted file mode 100644 index 8b337ef..0000000 --- a/examples/lookup_user_v2.py +++ /dev/null @@ -1,16 +0,0 @@ -# Documentation -# https://documenter.getpostman.com/view/9956214/T1LMiT5U#auth-info-784efcda-ed4c-4491-a4c0-a26470a67400 - -# You must opt-in to v2 here: -# https://developer.twitter.com/en/portal/opt-in - -from TwitterAPI import TwitterAPI, TwitterPager - -SCREEN_NAME = 'TheTweetOfGod' - -api = TwitterAPI(, - , - auth_type='oAuth2') - -r = api.request(f'users/by/username/:{SCREEN_NAME}', version='2') -print(r.json()['data']['id'] if r.status_code == 200 else 'PROBLEM: ' + r.text) diff --git a/examples/connect_using_login.py b/examples/v1.1/connect_using_login.py similarity index 100% rename from examples/connect_using_login.py rename to examples/v1.1/connect_using_login.py diff --git a/examples/connect_using_oauth.py b/examples/v1.1/connect_using_oauth.py similarity index 100% rename from examples/connect_using_oauth.py rename to examples/v1.1/connect_using_oauth.py diff --git a/examples/delete_last_tweet.py b/examples/v1.1/delete_last_tweet.py similarity index 100% rename from examples/delete_last_tweet.py rename to examples/v1.1/delete_last_tweet.py diff --git a/examples/direct_message.py b/examples/v1.1/direct_message.py similarity index 100% rename from examples/direct_message.py rename to examples/v1.1/direct_message.py diff --git a/examples/dump_timeline.py b/examples/v1.1/dump_timeline.py similarity index 100% rename from examples/dump_timeline.py rename to examples/v1.1/dump_timeline.py diff --git a/examples/logging.py b/examples/v1.1/logging.py similarity index 100% rename from examples/logging.py rename to examples/v1.1/logging.py diff --git a/examples/lookup_tweet.py b/examples/v1.1/lookup_tweet.py similarity index 100% rename from examples/lookup_tweet.py rename to examples/v1.1/lookup_tweet.py diff --git a/examples/lookup_user.py b/examples/v1.1/lookup_user.py similarity index 100% rename from examples/lookup_user.py rename to examples/v1.1/lookup_user.py diff --git a/examples/page_tweets.py b/examples/v1.1/page_tweets.py similarity index 100% rename from examples/page_tweets.py rename to examples/v1.1/page_tweets.py diff --git a/examples/post_tweet.py b/examples/v1.1/post_tweet.py similarity index 100% rename from examples/post_tweet.py rename to examples/v1.1/post_tweet.py diff --git a/examples/premium_search.py b/examples/v1.1/premium_search.py similarity index 100% rename from examples/premium_search.py rename to examples/v1.1/premium_search.py diff --git a/examples/sample_frequency.py b/examples/v1.1/sample_frequency.py similarity index 100% rename from examples/sample_frequency.py rename to examples/v1.1/sample_frequency.py diff --git a/examples/search_tweets.py b/examples/v1.1/search_tweets.py similarity index 100% rename from examples/search_tweets.py rename to examples/v1.1/search_tweets.py diff --git a/examples/stream_tweets.py b/examples/v1.1/stream_tweets.py similarity index 100% rename from examples/stream_tweets.py rename to examples/v1.1/stream_tweets.py diff --git a/examples/unfollow_friends.py b/examples/v1.1/unfollow_friends.py similarity index 100% rename from examples/unfollow_friends.py rename to examples/v1.1/unfollow_friends.py diff --git a/examples/upload_image.py b/examples/v1.1/upload_image.py similarity index 100% rename from examples/upload_image.py rename to examples/v1.1/upload_image.py diff --git a/examples/upload_tweet.py b/examples/v1.1/upload_tweet.py similarity index 100% rename from examples/upload_tweet.py rename to examples/v1.1/upload_tweet.py diff --git a/examples/upload_video.py b/examples/v1.1/upload_video.py similarity index 100% rename from examples/upload_video.py rename to examples/v1.1/upload_video.py diff --git a/examples/v2_SearchTweets.py b/examples/v2/lookup_tweet_v2.py similarity index 83% rename from examples/v2_SearchTweets.py rename to examples/v2/lookup_tweet_v2.py index 5b9e4a8..c1e2dc6 100644 --- a/examples/v2_SearchTweets.py +++ b/examples/v2/lookup_tweet_v2.py @@ -1,9 +1,11 @@ from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +TWEET_ID = '964575983633252353' + try: o = TwitterOAuth.read_file() api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') - r = api.request('tweets/search/recent', {'query':'pizza', 'tweet.fields':'public_metrics'}) + r = api.request(f'tweets/:{TWEET_ID}') for item in r: print(item) diff --git a/examples/v2/lookup_user_v2.py b/examples/v2/lookup_user_v2.py new file mode 100644 index 0000000..5232cdb --- /dev/null +++ b/examples/v2/lookup_user_v2.py @@ -0,0 +1,24 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError + +SCREEN_NAME = 'TheTweetOfGod' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'users/by/username/:{SCREEN_NAME}') + + for item in r: + print(item) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2_PageTweets.py b/examples/v2/page_tweets.py similarity index 85% rename from examples/v2_PageTweets.py rename to examples/v2/page_tweets.py index de7b36c..fadfcc8 100644 --- a/examples/v2_PageTweets.py +++ b/examples/v2/page_tweets.py @@ -1,9 +1,11 @@ from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager +QUERY = 'pizza' + try: o = TwitterOAuth.read_file() api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2'') - pager = TwitterPager(api, 'tweets/search/recent', {'query':'pizza'}) + pager = TwitterPager(api, 'tweets/search/recent', {'query':QUERY}) for item in pager.get_iterator(new_tweets=False): print(item) diff --git a/examples/v2/search_tweets.py b/examples/v2/search_tweets.py new file mode 100644 index 0000000..1b674d6 --- /dev/null +++ b/examples/v2/search_tweets.py @@ -0,0 +1,24 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError + +QUERY = 'pizza' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets/search/recent', {f'query':{QUERY}, 'tweet.fields':'public_metrics'}) + + for item in r: + print(item) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2_StreamTweets.py b/examples/v2/stream_tweets.py similarity index 96% rename from examples/v2_StreamTweets.py rename to examples/v2/stream_tweets.py index d01cfac..222a1ea 100644 --- a/examples/v2_StreamTweets.py +++ b/examples/v2/stream_tweets.py @@ -1,12 +1,14 @@ from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +QUERY = 'pizza' + try: o = TwitterOAuth.read_file() api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') # ADD STREAM RULES - r = api.request('tweets/search/stream/rules', {'add': [{'value':'pizza'}]}) + r = api.request('tweets/search/stream/rules', {'add': [{'value':QUERY}]}) print(f'[{r.status_code}] RULE ADDED: {r.text}') if r.status_code != 201: exit() From 174288fb9eb9c7714a9b8edec6dbd45faf7065e5 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Tue, 20 Oct 2020 17:39:26 -0400 Subject: [PATCH 049/139] examples fixes --- TwitterAPI/TwitterOAuth.py | 15 +++++- examples/cli/cli.py | 49 ++++++++++++++++--- .../{lookup_tweet_v2.py => lookup_tweet.py} | 0 .../v2/{lookup_user_v2.py => lookup_user.py} | 0 examples/v2/search_tweets.py | 2 +- 5 files changed, 55 insertions(+), 11 deletions(-) rename examples/v2/{lookup_tweet_v2.py => lookup_tweet.py} (100%) rename examples/v2/{lookup_user_v2.py => lookup_user.py} (100%) diff --git a/TwitterAPI/TwitterOAuth.py b/TwitterAPI/TwitterOAuth.py index 80953a6..1953827 100644 --- a/TwitterAPI/TwitterOAuth.py +++ b/TwitterAPI/TwitterOAuth.py @@ -50,10 +50,21 @@ def read_file(cls, file_name=None): oauth = {} for line in f: if '=' in line: - name, value = line.split('=', 1) - oauth[name.strip()] = value.strip() + tokens = line.strip().split('=', 1) + if len(tokens) != 2 or not tokens[1]: + raise Exception(cls.usage(file_name)) + oauth[tokens[0].strip()] = tokens[1].strip() return TwitterOAuth( oauth['consumer_key'], oauth['consumer_secret'], oauth['access_token_key'], oauth['access_token_secret']) + + @classmethod + def usage(cls, file_name): + return (f'Credentials file: {file_name}\n' + 'Expected format:\n' + 'consumer_key=YOUR_CONSUMER_KEY\n' + 'consumer_secret=YOUR_CONSUMER_SECRET\n' + 'access_token_key=YOUR_ACCESS_TOKEN\n' + 'access_token_secret=YOUR_ACCESS_TOKEN_SECRET\n') \ No newline at end of file diff --git a/examples/cli/cli.py b/examples/cli/cli.py index 4ad8d0b..d3d9245 100644 --- a/examples/cli/cli.py +++ b/examples/cli/cli.py @@ -29,12 +29,12 @@ """ -__author__ = "geduldig" +__author__ = "Jonas Geduldig" __date__ = "June 7, 2013" __license__ = "MIT" -from TwitterAPI import TwitterAPI, __version__ +from TwitterAPI import TwitterAPI, TwitterRequestError, TwitterConnectionError, __version__ from TwitterAPI.TwitterOAuth import TwitterOAuth import argparse import codecs @@ -42,6 +42,9 @@ import sys +DEFAULT_API_VERSION = '1.1' + + # print UTF-8 to the console if sys.platform == "win32": from Unicode_win32 import stdout @@ -75,7 +78,7 @@ def _to_dict(param_list): if __name__ == '__main__': - print('TwitterAPI %s by geduldig' % __version__) + print('TwitterAPI %s by Jonas Geduldig' % __version__) parser = argparse.ArgumentParser( description='Request any Twitter Streaming or REST API endpoint') @@ -84,6 +87,13 @@ def _to_dict(param_list): metavar='FILENAME', type=str, help='file containing OAuth credentials') + parser.add_argument( + '-bearertoken', + metavar='BEARERTOKEN', + type=eval, + choices=[True, False], + default=False, + help='use OAuth 2.0') parser.add_argument( '-endpoint', metavar='ENDPOINT', @@ -108,27 +118,50 @@ def _to_dict(param_list): type=int, help='number of spaces to indent json output', default=None) + parser.add_argument( + '-version', + metavar='VERSION', + type=str, + help='Twitter API version', + default=DEFAULT_API_VERSION) + parser.add_argument( + '-methodoverride', + metavar='METHOD', + type=str, + help='override default HTTP method', + default=None) args = parser.parse_args() try: params = _to_dict(args.parameters) oauth = TwitterOAuth.read_file(args.oauth) + auth_type = 'oAuth2' if args.bearertoken else 'oAuth1' api = TwitterAPI(oauth.consumer_key, - oauth.consumer_secret, - oauth.access_token_key, - oauth.access_token_secret) - response = api.request(args.endpoint, params) + oauth.consumer_secret, + oauth.access_token_key, + oauth.access_token_secret, + api_version=args.version, + auth_type=auth_type) + response = api.request(args.endpoint, params, method_override=args.methodoverride) for item in response.get_iterator(): if not args.fields: - print( json.dumps(item, ensure_ascii='False', indent=args.indent)) + print(json.dumps(item, ensure_ascii='False', indent=args.indent)) else: for name in args.fields: value = _search(name, item) if value: print('%s: %s' % (name, value)) + except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + + except TwitterConnectionError as e: + print(e) + except KeyboardInterrupt: print('Terminated by user') diff --git a/examples/v2/lookup_tweet_v2.py b/examples/v2/lookup_tweet.py similarity index 100% rename from examples/v2/lookup_tweet_v2.py rename to examples/v2/lookup_tweet.py diff --git a/examples/v2/lookup_user_v2.py b/examples/v2/lookup_user.py similarity index 100% rename from examples/v2/lookup_user_v2.py rename to examples/v2/lookup_user.py diff --git a/examples/v2/search_tweets.py b/examples/v2/search_tweets.py index 1b674d6..11e58f8 100644 --- a/examples/v2/search_tweets.py +++ b/examples/v2/search_tweets.py @@ -5,7 +5,7 @@ try: o = TwitterOAuth.read_file() api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') - r = api.request(f'tweets/search/recent', {f'query':{QUERY}, 'tweet.fields':'public_metrics'}) + r = api.request('tweets/search/recent', {'query':QUERY, 'tweet.fields':'public_metrics'}) for item in r: print(item) From b62acc76c2ca8658b4f37771c3316ce6b510e1db Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Tue, 20 Oct 2020 17:40:00 -0400 Subject: [PATCH 050/139] examples fixes --- TwitterAPI/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 457279b..d92db32 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.0.1' +__version__ = '2.6.0.2' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From f31e32e3c242426ce56f3d9c3f70c5ed9ef284d4 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Wed, 21 Oct 2020 11:29:43 -0400 Subject: [PATCH 051/139] added more v2 examples --- TwitterAPI/__init__.py | 2 +- examples/v2/conversation.py | 30 ++++++++++++++++++++++++++++++ examples/v2/expansions_media.py | 31 +++++++++++++++++++++++++++++++ examples/v2/expansions_place.py | 31 +++++++++++++++++++++++++++++++ examples/v2/expansions_poll.py | 31 +++++++++++++++++++++++++++++++ examples/v2/expansions_tweet.py | 31 +++++++++++++++++++++++++++++++ examples/v2/expansions_user.py | 31 +++++++++++++++++++++++++++++++ 7 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 examples/v2/conversation.py create mode 100644 examples/v2/expansions_media.py create mode 100644 examples/v2/expansions_place.py create mode 100644 examples/v2/expansions_poll.py create mode 100644 examples/v2/expansions_tweet.py create mode 100644 examples/v2/expansions_user.py diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index d92db32..d7f9f58 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.0.2' +__version__ = '2.6.0.3' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/examples/v2/conversation.py b/examples/v2/conversation.py new file mode 100644 index 0000000..dae4f2f --- /dev/null +++ b/examples/v2/conversation.py @@ -0,0 +1,30 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager +import json + +CONVERSION_ID = '1318769013640450048' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + pager = TwitterPager(api, 'tweets/search/recent', + { + 'query':f'conversation_id:{CONVERSION_ID}', + 'tweet.fields':'conversation_id' + }) + + # wait=2 means wait 2 seconds between each request. + # The rate limit is 450 request per 15 minutes, or + # 15*60/450 = 2 seconds + for item in pager.get_iterator(wait=2): + print(json.dumps(item, indent=2)) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2/expansions_media.py b/examples/v2/expansions_media.py new file mode 100644 index 0000000..2db13d2 --- /dev/null +++ b/examples/v2/expansions_media.py @@ -0,0 +1,31 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +TWEET_ID = '1318585013944942593' +EXPANSIONS = 'attachments.media_keys' +MEDIA_FIELDS = 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets/:{TWEET_ID}', + { + 'expansions':EXPANSIONS, + 'media.fields':MEDIA_FIELDS + }) + + for item in r: + print(json.dumps(item, indent=2)) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2/expansions_place.py b/examples/v2/expansions_place.py new file mode 100644 index 0000000..671bd49 --- /dev/null +++ b/examples/v2/expansions_place.py @@ -0,0 +1,31 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +TWEET_ID = '1295386980495364101' +EXPANSIONS = 'geo.place_id' +PLACE_FIELDS = 'contained_within,country,country_code,full_name,geo,id,name,place_type' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets/:{TWEET_ID}', + { + 'expansions':EXPANSIONS, + 'place.fields':PLACE_FIELDS + }) + + for item in r: + print(json.dumps(item, indent=2)) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2/expansions_poll.py b/examples/v2/expansions_poll.py new file mode 100644 index 0000000..72b9754 --- /dev/null +++ b/examples/v2/expansions_poll.py @@ -0,0 +1,31 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +TWEET_ID = '1317617296978833408' +EXPANSIONS = 'attachments.poll_ids' +POLL_FIELDS = 'duration_minutes,end_datetime,id,options,voting_status' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets/:{TWEET_ID}', + { + 'expansions':EXPANSIONS, + 'poll.fields':POLL_FIELDS + }) + + for item in r: + print(json.dumps(item, indent=2)) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2/expansions_tweet.py b/examples/v2/expansions_tweet.py new file mode 100644 index 0000000..0f125a4 --- /dev/null +++ b/examples/v2/expansions_tweet.py @@ -0,0 +1,31 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +TWEET_ID = '1275914014867050499' +EXPANSIONS = 'referenced_tweets.id' +TWEET_FIELDS = 'attachments,author_id,context_annotations,conversation_id,created_at,entities,geo,id,in_reply_to_user_id,lang,public_metrics,text' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets/:{TWEET_ID}', + { + 'expansions':EXPANSIONS, + 'tweet.fields':TWEET_FIELDS + }) + + for item in r: + print(json.dumps(item, indent=2)) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2/expansions_user.py b/examples/v2/expansions_user.py new file mode 100644 index 0000000..70f43ce --- /dev/null +++ b/examples/v2/expansions_user.py @@ -0,0 +1,31 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +TWEET_ID = '1275914014867050499' +EXPANSIONS = 'author_id,in_reply_to_user_id,referenced_tweets.id.author_id' +USER_FIELDS = 'location,profile_image_url,verified' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets/:{TWEET_ID}', + { + 'expansions':EXPANSIONS, + 'user.fields':USER_FIELDS + }) + + for item in r: + print(json.dumps(item, indent=2)) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file From ccbca7b20a9589cd3a7b95e3c5219b1bbd766d8c Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 23 Oct 2020 10:28:51 -0400 Subject: [PATCH 052/139] Fix TwitterPager, add conversation example --- CHANGES.txt => CHANGE.log | 4 +- TwitterAPI/TwitterAPI.py | 7 ++- TwitterAPI/TwitterPager.py | 18 ++++--- TwitterAPI/__init__.py | 2 +- examples/v2/conversation.py | 30 ----------- examples/v2/conversation_tree.py | 89 ++++++++++++++++++++++++++++++++ examples/v2/lookup_tweet.py | 3 +- examples/v2/lookup_user.py | 2 +- examples/v2/page_tweets.py | 4 +- 9 files changed, 113 insertions(+), 46 deletions(-) rename CHANGES.txt => CHANGE.log (97%) delete mode 100644 examples/v2/conversation.py create mode 100644 examples/v2/conversation_tree.py diff --git a/CHANGES.txt b/CHANGE.log similarity index 97% rename from CHANGES.txt rename to CHANGE.log index 1223303..aec3255 100644 --- a/CHANGES.txt +++ b/CHANGE.log @@ -147,4 +147,6 @@ v2.5.14, 17 Sep 2020 -- Fix TwitterPager to iterate integer ids. v2.6.0, 04 Oct 2020 -- Merged dylancaponi-twitter-v2 with V2 support. Made several modifications to complete V2 support. - Dropping Python 2.7 compatibility. \ No newline at end of file + Dropping Python 2.7 compatibility. + +v2.6.1, 23 Oct 2020 -- Fix TwitterPager, add conversation example. \ No newline at end of file diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 44bb2a7..3f2f597 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -309,8 +309,11 @@ def __init__(self, response): self.results = resp['ids'] elif 'results' in resp: self.results = resp['results'] - elif 'data' in resp and not isinstance(resp['data'], dict): - self.results = resp['data'] + elif 'data' in resp: + if not isinstance(resp['data'], dict): + self.results = resp['data'] + else: + self.results = [resp['data']] elif hasattr(resp, '__iter__') and not isinstance(resp, dict): if len(resp) > 0 and 'trends' in resp[0]: self.results = resp[0]['trends'] diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py index 554d0c9..d1fa5bb 100644 --- a/TwitterAPI/TwitterPager.py +++ b/TwitterAPI/TwitterPager.py @@ -62,7 +62,7 @@ def get_iterator(self, wait=5, new_tweets=False): raise TwitterConnectionError(item) yield item - json = r.json() + data = r.json() # CHECK FOR NEXT PAGE OR BAIL... if self.api.version == '1.1': @@ -70,16 +70,16 @@ def get_iterator(self, wait=5, new_tweets=False): # otherwise, use id to get next page is_premium_search = self.params and 'query' in self.params and self.api.version == '1.1' cursor = -1 - if new_tweets and 'previous_cursor' in json: - cursor = json['previous_cursor'] + if new_tweets and 'previous_cursor' in data: + cursor = data['previous_cursor'] cursor_param = 'cursor' elif not new_tweets: - if 'next_cursor' in json: - cursor = json['next_cursor'] + if 'next_cursor' in data: + cursor = data['next_cursor'] cursor_param = 'cursor' - elif 'next' in json: + elif 'next' in data: # 'next' is used by Premium Search (OLD searches only) - cursor = json['next'] + cursor = data['next'] cursor_param = 'next' # bail when no more results @@ -90,7 +90,9 @@ def get_iterator(self, wait=5, new_tweets=False): elif not new_tweets and item_count == 0: break else: # VERSION 2 - meta = json['meta'] + meta = data['meta'] + if not new_tweets and not 'next_token' in meta: + break # SLEEP... elapsed = time.time() - start diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index d7f9f58..1a23b19 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.0.3' +__version__ = '2.6.1' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/examples/v2/conversation.py b/examples/v2/conversation.py deleted file mode 100644 index dae4f2f..0000000 --- a/examples/v2/conversation.py +++ /dev/null @@ -1,30 +0,0 @@ -from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager -import json - -CONVERSION_ID = '1318769013640450048' - -try: - o = TwitterOAuth.read_file() - api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') - pager = TwitterPager(api, 'tweets/search/recent', - { - 'query':f'conversation_id:{CONVERSION_ID}', - 'tweet.fields':'conversation_id' - }) - - # wait=2 means wait 2 seconds between each request. - # The rate limit is 450 request per 15 minutes, or - # 15*60/450 = 2 seconds - for item in pager.get_iterator(wait=2): - print(json.dumps(item, indent=2)) - -except TwitterRequestError as e: - print(e.status_code) - for msg in iter(e): - print(msg) - -except TwitterConnectionError as e: - print(e) - -except Exception as e: - print(e) \ No newline at end of file diff --git a/examples/v2/conversation_tree.py b/examples/v2/conversation_tree.py new file mode 100644 index 0000000..3b982e3 --- /dev/null +++ b/examples/v2/conversation_tree.py @@ -0,0 +1,89 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager +import json + +CONVERSATION_ID = '1318769013640450048' + +class Node: + def __init__(self, data): + """data is a tweet's json object""" + self.data = data + self.children = [] + + def id(self): + """a node is identified by its author""" + return self.data['author_id'] + + def reply_to(self): + """the reply-to user is the parent of the node""" + return self.data['in_reply_to_user_id'] + + def print_tree(self, level): + """level is 0 for the root node, then incremented for every generaion""" + print(f'{level*"_"}{level}: {self.id()}') + level += 1 + for child in self.children: + child.print_tree(level) + + def find_parent_of(self, node): + """append a node to the children of it's reply-to user""" + if node.reply_to() == self.id(): + self.children.append(node) + return True + for child in self.children: + if child.find_parent_of(node): + return True + return False + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + + # GET ROOT OF THE CONVERSATION + + r = api.request(f'tweets/:{CONVERSATION_ID}', + { + 'tweet.fields':'author_id,conversation_id,created_at,in_reply_to_user_id' + }) + + for item in r: + root = Node(item) + print(f'ROOT {root.id()}') + + # GET ALL REPLIES IN CONVERSATION + # (RETURNED IN REVERSE CHRONOLOGICAL ORDER) + + pager = TwitterPager(api, 'tweets/search/recent', + { + 'query':f'conversation_id:{CONVERSATION_ID}', + 'tweet.fields':'author_id,conversation_id,created_at,in_reply_to_user_id' + }) + + # "wait=2" means wait 2 seconds between each request. + # The rate limit is 450 requests per 15 minutes, or + # 15*60/450 = 2 seconds. + + orphans = [] + + for item in pager.get_iterator(wait=2): + node = Node(item) + print(f'{node.id()} => {node.reply_to()}') + # REMOVE ORPHANS THAT ARE CHILDREN OF THE NODE + orphans = [orphan for orphan in orphans if not node.find_parent_of(orphan)] + # IF NODE CANNOT BE PLACED IN CURRENT TREE, ADD NODE TO ORPHANS + if not root.find_parent_of(node): + orphans.append(node) + + print('\nTREE...') + root.print_tree(0) + assert len(orphans) == 0, f'{len(orphans)} orphaned tweets' + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2/lookup_tweet.py b/examples/v2/lookup_tweet.py index c1e2dc6..a8ae641 100644 --- a/examples/v2/lookup_tweet.py +++ b/examples/v2/lookup_tweet.py @@ -2,6 +2,7 @@ TWEET_ID = '964575983633252353' + try: o = TwitterOAuth.read_file() api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') @@ -21,4 +22,4 @@ print(e) except Exception as e: - print(e) \ No newline at end of file + print(e) diff --git a/examples/v2/lookup_user.py b/examples/v2/lookup_user.py index 5232cdb..752215a 100644 --- a/examples/v2/lookup_user.py +++ b/examples/v2/lookup_user.py @@ -21,4 +21,4 @@ print(e) except Exception as e: - print(e) \ No newline at end of file + print(e) diff --git a/examples/v2/page_tweets.py b/examples/v2/page_tweets.py index fadfcc8..f61e88e 100644 --- a/examples/v2/page_tweets.py +++ b/examples/v2/page_tweets.py @@ -4,7 +4,7 @@ try: o = TwitterOAuth.read_file() - api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2'') + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') pager = TwitterPager(api, 'tweets/search/recent', {'query':QUERY}) for item in pager.get_iterator(new_tweets=False): print(item) @@ -18,4 +18,4 @@ print(e) except Exception as e: - print(e) \ No newline at end of file + print(e) From dae214cdbf9bc38b5412a2736a718bc90fb0e05f Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 23 Oct 2020 12:51:27 -0400 Subject: [PATCH 053/139] conversation_tree example update --- examples/v2/conversation_tree.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/examples/v2/conversation_tree.py b/examples/v2/conversation_tree.py index 3b982e3..700f205 100644 --- a/examples/v2/conversation_tree.py +++ b/examples/v2/conversation_tree.py @@ -1,9 +1,8 @@ from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager -import json CONVERSATION_ID = '1318769013640450048' -class Node: +class TreeNode: def __init__(self, data): """data is a tweet's json object""" self.data = data @@ -17,13 +16,6 @@ def reply_to(self): """the reply-to user is the parent of the node""" return self.data['in_reply_to_user_id'] - def print_tree(self, level): - """level is 0 for the root node, then incremented for every generaion""" - print(f'{level*"_"}{level}: {self.id()}') - level += 1 - for child in self.children: - child.print_tree(level) - def find_parent_of(self, node): """append a node to the children of it's reply-to user""" if node.reply_to() == self.id(): @@ -34,6 +26,13 @@ def find_parent_of(self, node): return True return False + def print_tree(self, level): + """level is 0 for the root node, then incremented for every generation""" + print(f'{level*"_"}{level}: {self.id()}') + level += 1 + for child in self.children: + child.print_tree(level) + try: o = TwitterOAuth.read_file() api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') @@ -46,7 +45,7 @@ def find_parent_of(self, node): }) for item in r: - root = Node(item) + root = TreeNode(item) print(f'ROOT {root.id()}') # GET ALL REPLIES IN CONVERSATION @@ -65,11 +64,11 @@ def find_parent_of(self, node): orphans = [] for item in pager.get_iterator(wait=2): - node = Node(item) + node = TreeNode(item) print(f'{node.id()} => {node.reply_to()}') - # REMOVE ORPHANS THAT ARE CHILDREN OF THE NODE + # COLLECT ANY ORPHANS THAT ARE NODE'S CHILD orphans = [orphan for orphan in orphans if not node.find_parent_of(orphan)] - # IF NODE CANNOT BE PLACED IN CURRENT TREE, ADD NODE TO ORPHANS + # IF NODE CANNOT BE PLACED IN TREE, ORPHAN IT UNTIL IT'S PARENT IS FOUND if not root.find_parent_of(node): orphans.append(node) From 8824cdd6150ec97c11ae12c33aa3c47873d8c1a9 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 23 Oct 2020 12:52:07 -0400 Subject: [PATCH 054/139] conversation_tree example update --- TwitterAPI/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 1a23b19..de04204 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.1' +__version__ = '2.6.1.1' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From bb51a4eca11f3a8c0839a012e8ca97fcadf8aebb Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 23 Oct 2020 12:54:39 -0400 Subject: [PATCH 055/139] conversation_tree example update --- TwitterAPI/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index de04204..54308d0 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.1.1' +__version__ = '2.6.1.2' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 4f20eab3ac600ab4e497b103fa727576a148041a Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 23 Oct 2020 12:58:49 -0400 Subject: [PATCH 056/139] conversation_tree example update --- TwitterAPI/__init__.py | 2 +- examples/v2/conversation_tree.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 54308d0..b89e763 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.1.2' +__version__ = '2.6.1.3' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/examples/v2/conversation_tree.py b/examples/v2/conversation_tree.py index 700f205..4be704b 100644 --- a/examples/v2/conversation_tree.py +++ b/examples/v2/conversation_tree.py @@ -68,7 +68,7 @@ def print_tree(self, level): print(f'{node.id()} => {node.reply_to()}') # COLLECT ANY ORPHANS THAT ARE NODE'S CHILD orphans = [orphan for orphan in orphans if not node.find_parent_of(orphan)] - # IF NODE CANNOT BE PLACED IN TREE, ORPHAN IT UNTIL IT'S PARENT IS FOUND + # IF NODE CANNOT BE PLACED IN TREE, ORPHAN IT UNTIL ITS PARENT IS FOUND if not root.find_parent_of(node): orphans.append(node) From 5f116194ce05b8a3911b3b4a1113678789fca124 Mon Sep 17 00:00:00 2001 From: Carl Kovamees Date: Mon, 26 Oct 2020 22:13:32 -0400 Subject: [PATCH 057/139] Added subtitles API calls --- TwitterAPI/constants.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index f897cf6..45073f8 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -110,7 +110,9 @@ 'lists/update': ('POST', 'api'), 'media/metadata/create': ('POST', 'upload'), - 'media/upload': ('POST', 'upload'), + 'media/upload': ('POST', 'upload'), + 'media/subtitles/create': ('POST', 'upload'), + 'media/subtitles/delete': ('POST', 'upload'), 'mutes/users/create': ('POST', 'api'), 'mutes/users/destroy': ('POST', 'api'), From 6f4c79d728fe9f6b144c2aa33ceeff44fcaa86fe Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Tue, 27 Oct 2020 11:02:40 -0400 Subject: [PATCH 058/139] Merged ckovamees-master --- CHANGE.log | 4 +++- TwitterAPI/__init__.py | 2 +- TwitterAPI/constants.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index aec3255..e565749 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -149,4 +149,6 @@ v2.6.0, 04 Oct 2020 -- Merged dylancaponi-twitter-v2 with V2 support. Made several modifications to complete V2 support. Dropping Python 2.7 compatibility. -v2.6.1, 23 Oct 2020 -- Fix TwitterPager, add conversation example. \ No newline at end of file +v2.6.1, 23 Oct 2020 -- Fix TwitterPager, add conversation example. + +v2.6.2, 27 Oct 2020 -- Merged ckovamees-master with add media endpoints. diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index b89e763..1d482a9 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.1.3' +__version__ = '2.6.2' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 45073f8..39cf1ae 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -110,7 +110,7 @@ 'lists/update': ('POST', 'api'), 'media/metadata/create': ('POST', 'upload'), - 'media/upload': ('POST', 'upload'), + 'media/upload': ('POST', 'upload'), # use method_override for 'GET' 'media/subtitles/create': ('POST', 'upload'), 'media/subtitles/delete': ('POST', 'upload'), From 4d5c636aea15940907d021b1f6b7a8dc382ae2ad Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 30 Oct 2020 08:41:17 -0400 Subject: [PATCH 059/139] added more v2 examples --- TwitterAPI/TwitterAPI.py | 8 -------- TwitterAPI/__init__.py | 2 +- examples/v2/conversation_tree.py | 2 +- examples/v2/rules_add.py | 24 +++++++++++++++++++++++ examples/v2/rules_del.py | 27 ++++++++++++++++++++++++++ examples/v2/rules_del_all.py | 33 ++++++++++++++++++++++++++++++++ examples/v2/rules_get.py | 22 +++++++++++++++++++++ 7 files changed, 108 insertions(+), 10 deletions(-) create mode 100644 examples/v2/rules_add.py create mode 100644 examples/v2/rules_del.py create mode 100644 examples/v2/rules_del_all.py create mode 100644 examples/v2/rules_get.py diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 3f2f597..1701174 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -168,11 +168,6 @@ def request(self, resource, params=None, files=None, method_override=None, api_v else: session.stream = False timeout = self.REST_TIMEOUT - # if method == 'POST': - # data = params - # params = None - # else: - # data = None d = p = j = None if method == 'POST': if api_version == '1.1': @@ -195,9 +190,6 @@ def request(self, resource, params=None, files=None, method_override=None, api_v r = session.request( method, url, - # data=data, - # params=params, - # json=params, data=d, params=p, json=j, diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 1d482a9..3bd74d9 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.2' +__version__ = '2.6.2.1' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/examples/v2/conversation_tree.py b/examples/v2/conversation_tree.py index 4be704b..2c0cda9 100644 --- a/examples/v2/conversation_tree.py +++ b/examples/v2/conversation_tree.py @@ -27,7 +27,7 @@ def find_parent_of(self, node): return False def print_tree(self, level): - """level is 0 for the root node, then incremented for every generation""" + """level 0 is the root node, then incremented for subsequent generations""" print(f'{level*"_"}{level}: {self.id()}') level += 1 for child in self.children: diff --git a/examples/v2/rules_add.py b/examples/v2/rules_add.py new file mode 100644 index 0000000..733133e --- /dev/null +++ b/examples/v2/rules_add.py @@ -0,0 +1,24 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +QUERY = 'pizza"' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + + # ADD STREAM RULES + + r = api.request('tweets/search/stream/rules', {'add': [{'value':QUERY}]}) + print(f'[{r.status_code}] RULE ADDED: {json.dumps(r.json(), indent=2)}\n') + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/rules_del.py b/examples/v2/rules_del.py new file mode 100644 index 0000000..752cc93 --- /dev/null +++ b/examples/v2/rules_del.py @@ -0,0 +1,27 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +RULE_IDS = [ + "1317935598431309824", + "1321813278532751361", +] + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + + # DELETE STREAM RULES + + r = api.request('tweets/search/stream/rules', {'delete': {'ids':RULE_IDS}}) + print(f'[{r.status_code}] RULES DELETED: {json.dumps(r.json(), indent=2)}\n') + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/rules_del_all.py b/examples/v2/rules_del_all.py new file mode 100644 index 0000000..5e64748 --- /dev/null +++ b/examples/v2/rules_del_all.py @@ -0,0 +1,33 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + + # GET STREAM RULES + + rule_ids = [] + r = api.request('tweets/search/stream/rules', method_override='GET') + for item in r: + if 'id' in item: + rule_ids.append(item['id']) + else: + print(json.dumps(item, indent=2)) + + # DELETE STREAM RULES + + if len(rule_ids) > 0: + r = api.request('tweets/search/stream/rules', {'delete': {'ids':rule_ids}}) + print(f'[{r.status_code}] RULES DELETED: {json.dumps(r.json(), indent=2)}\n') + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/rules_get.py b/examples/v2/rules_get.py new file mode 100644 index 0000000..1a0c80d --- /dev/null +++ b/examples/v2/rules_get.py @@ -0,0 +1,22 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + + # GET STREAM RULES + + r = api.request('tweets/search/stream/rules', method_override='GET') + print(f'[{r.status_code}] RULES: {json.dumps(r.json(), indent=2)}\n') + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) From 2fdb4a4e33967ac66a9905fdd0dfb2efc3ab4da9 Mon Sep 17 00:00:00 2001 From: peguerosdc Date: Tue, 5 Jan 2021 01:08:12 -0600 Subject: [PATCH 060/139] Adding user's followers/following v2 --- TwitterAPI/constants.py | 2 ++ examples/v2/user_followers_following.py | 31 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 examples/v2/user_followers_following.py diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 39cf1ae..4a39bb0 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -245,6 +245,8 @@ 'tweets/search/stream/rules': ('POST', 'api'), # use method_override for 'GET' 'users': ('GET', 'api'), 'users/:PARAM': ('GET', 'api'), # ID + 'users/:PARAM/followers': ('GET', 'api'), # ID + 'users/:PARAM/following': ('GET', 'api'), # ID 'users/by': ('GET', 'api'), 'users/by/username/:PARAM': ('GET', 'api'), # USERNAME } diff --git a/examples/v2/user_followers_following.py b/examples/v2/user_followers_following.py new file mode 100644 index 0000000..efa1df7 --- /dev/null +++ b/examples/v2/user_followers_following.py @@ -0,0 +1,31 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError + +USER_ID = 'id' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + + # Get followers + r = api.request(f'users/:{USER_ID}/followers') + followers = r.json()['data'] + for f in followers: + print(f) + + # Get following + for item in r: + r = api.request(f'users/:{USER_ID}/following') + following = r.json()['data'] + for f in following: + print(f) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) From 2f224de7f9f00ddfb5b8e6cd6dfc604fbcf6db24 Mon Sep 17 00:00:00 2001 From: peguerosdc Date: Tue, 5 Jan 2021 11:47:05 -0600 Subject: [PATCH 061/139] Using iterator in followers/following example --- examples/v2/user_followers_following.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/examples/v2/user_followers_following.py b/examples/v2/user_followers_following.py index efa1df7..e55317c 100644 --- a/examples/v2/user_followers_following.py +++ b/examples/v2/user_followers_following.py @@ -7,15 +7,12 @@ api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') # Get followers - r = api.request(f'users/:{USER_ID}/followers') - followers = r.json()['data'] + followers = api.request(f'users/:{USER_ID}/followers') for f in followers: print(f) # Get following - for item in r: - r = api.request(f'users/:{USER_ID}/following') - following = r.json()['data'] + following = api.request(f'users/:{USER_ID}/following') for f in following: print(f) From 4c97c39a95a926f817a594582fa295c708ffcfab Mon Sep 17 00:00:00 2001 From: peguerosdc Date: Tue, 5 Jan 2021 22:46:40 -0600 Subject: [PATCH 062/139] Adding support to GET users/:PARAM/tweets --- TwitterAPI/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 4a39bb0..29ddb99 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -247,6 +247,7 @@ 'users/:PARAM': ('GET', 'api'), # ID 'users/:PARAM/followers': ('GET', 'api'), # ID 'users/:PARAM/following': ('GET', 'api'), # ID + 'users/:PARAM/tweets': ('GET', 'api'), # ID 'users/by': ('GET', 'api'), 'users/by/username/:PARAM': ('GET', 'api'), # USERNAME } From 8b0f1e9fb4c3acdd7b9d0d482010fbc6a3f22b26 Mon Sep 17 00:00:00 2001 From: peguerosdc Date: Sat, 9 Jan 2021 18:44:21 -0600 Subject: [PATCH 063/139] Following/followers example using TwitterDev account --- examples/v2/user_followers_following.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/v2/user_followers_following.py b/examples/v2/user_followers_following.py index e55317c..e3a0db8 100644 --- a/examples/v2/user_followers_following.py +++ b/examples/v2/user_followers_following.py @@ -1,6 +1,6 @@ from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError -USER_ID = 'id' +USER_ID = '2244994945' # https://twitter.com/TwitterDev try: o = TwitterOAuth.read_file() From 68ff944e6304442693753cae2074a556de015b1f Mon Sep 17 00:00:00 2001 From: peguerosdc Date: Sat, 9 Jan 2021 18:44:52 -0600 Subject: [PATCH 064/139] Adding example of 2/users/:id/tweets --- examples/v2/user_tweets.py | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 examples/v2/user_tweets.py diff --git a/examples/v2/user_tweets.py b/examples/v2/user_tweets.py new file mode 100644 index 0000000..63ce81f --- /dev/null +++ b/examples/v2/user_tweets.py @@ -0,0 +1,43 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +from pprint import pprint + +USER_ID = '2244994945' # https://twitter.com/TwitterDev + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + + # Get tweets - default setting + tweets = api.request(f'users/:{USER_ID}/tweets') + for t in tweets: + print(t) + + # Get tweets with customization - (5 tweets only with created_at timestamp) + print() + params = {'max_results': 5, 'tweet.fields': 'created_at'} + tweets = api.request(f'users/:{USER_ID}/tweets', params) + for t in tweets: + pprint(t) + + # Get next 5 tweets + print() + next_token = tweets.json()['meta']['next_token'] + params = {'max_results': 5, 'tweet.fields': 'created_at', 'pagination_token': next_token} + tweets = api.request(f'users/:{USER_ID}/tweets', params) + for t in tweets: + pprint(t) + + +except TwitterRequestError as e: + print('Request error') + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print('Connection error') + print(e) + +except Exception as e: + print('Exception') + print(e) \ No newline at end of file From a9ce466244fd9e9605f50c3b51ea3b410723d245 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Tue, 12 Jan 2021 22:18:12 -0500 Subject: [PATCH 065/139] merged peguerosdc-master --- CHANGE.log | 2 ++ TwitterAPI/TwitterAPI.py | 26 +++++++++----------------- TwitterAPI/__init__.py | 2 +- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index e565749..571d7c5 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -152,3 +152,5 @@ v2.6.0, 04 Oct 2020 -- Merged dylancaponi-twitter-v2 with V2 support. v2.6.1, 23 Oct 2020 -- Fix TwitterPager, add conversation example. v2.6.2, 27 Oct 2020 -- Merged ckovamees-master with add media endpoints. + +v2.6.3, 12 Jan 2021 -- Merged peguerosdc-master with v2 user endpoints. \ No newline at end of file diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 1701174..e6dfa38 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -179,23 +179,15 @@ def request(self, resource, params=None, files=None, method_override=None, api_v else: p = params try: - if False and method == 'PUT': - session.headers['Content-type'] = 'application/json' - data = params - r = session.request( - method, - url, - json=data) - else: - r = session.request( - method, - url, - data=d, - params=p, - json=j, - timeout=(self.CONNECTION_TIMEOUT, timeout), - files=files, - proxies=self.proxies) + r = session.request( + method, + url, + data=d, + params=p, + json=j, + timeout=(self.CONNECTION_TIMEOUT, timeout), + files=files, + proxies=self.proxies) except (ConnectionError, ProtocolError, ReadTimeout, ReadTimeoutError, SSLError, ssl.SSLError, socket.error) as e: raise TwitterConnectionError(e) diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 3bd74d9..848de57 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.2.1' +__version__ = '2.6.3' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 47c1dc069fc566fcf01a16372aaab9ce617115e6 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Wed, 27 Jan 2021 09:11:56 -0500 Subject: [PATCH 066/139] fixed converstion_tree.py example --- CHANGE.log | 4 +++- TwitterAPI/__init__.py | 2 +- TwitterAPI/constants.py | 3 +++ examples/v2/conversation_tree.py | 31 +++++++++++++++++++------------ examples/v2/search_tweets.py | 9 ++++++++- 5 files changed, 34 insertions(+), 15 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index 571d7c5..645ab76 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -153,4 +153,6 @@ v2.6.1, 23 Oct 2020 -- Fix TwitterPager, add conversation example. v2.6.2, 27 Oct 2020 -- Merged ckovamees-master with add media endpoints. -v2.6.3, 12 Jan 2021 -- Merged peguerosdc-master with v2 user endpoints. \ No newline at end of file +v2.6.3, 12 Jan 2021 -- Merged peguerosdc-master with v2 user endpoints. + +v2.6.4, 27 Jan 2021 -- Fixed conversation_tree.py example. diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 848de57..f682c81 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.3' +__version__ = '2.6.4' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 29ddb99..198ea2b 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -239,7 +239,10 @@ 'tweets': ('GET', 'api'), 'tweets/:PARAM/hidden': ('PUT', 'api'), # ID 'tweets/:PARAM': ('GET', 'api'), # ID + 'tweets/compliance/jobs' ('POST', 'api'), # use method_override for 'GET' + 'tweets/compliance/jobs/:PARAM' ('POST', 'api'), # ID 'tweets/sample/stream': ('GET', 'api'), + 'tweets/search/all': ('GET', 'api'), 'tweets/search/recent': ('GET', 'api'), 'tweets/search/stream': ('GET', 'api'), 'tweets/search/stream/rules': ('POST', 'api'), # use method_override for 'GET' diff --git a/examples/v2/conversation_tree.py b/examples/v2/conversation_tree.py index 2c0cda9..8cec7dc 100644 --- a/examples/v2/conversation_tree.py +++ b/examples/v2/conversation_tree.py @@ -1,23 +1,30 @@ from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager +from datetime import datetime -CONVERSATION_ID = '1318769013640450048' +CONVERSATION_ID = '1354097088179687425' class TreeNode: def __init__(self, data): """data is a tweet's json object""" self.data = data self.children = [] + self.parent = None + if 'referenced_tweets' in self.data: + for tweet in self.data['referenced_tweets']: + if tweet['type'] == 'replied_to': + self.parent = tweet['id'] + break def id(self): - """a node is identified by its author""" - return self.data['author_id'] + """a node is identified by its tweet id""" + return self.data['id'] def reply_to(self): - """the reply-to user is the parent of the node""" - return self.data['in_reply_to_user_id'] + """the reply-to tweet is the parent of the node""" + return self.parent def find_parent_of(self, node): - """append a node to the children of it's reply-to user""" + """append a node to the children of it's parent tweet""" if node.reply_to() == self.id(): self.children.append(node) return True @@ -28,9 +35,9 @@ def find_parent_of(self, node): def print_tree(self, level): """level 0 is the root node, then incremented for subsequent generations""" - print(f'{level*"_"}{level}: {self.id()}') + print(f'{level*"_"}{level}: [{self.data["created_at"]}] {self.data["text"][0:40]}') level += 1 - for child in self.children: + for child in reversed(self.children): child.print_tree(level) try: @@ -41,7 +48,7 @@ def print_tree(self, level): r = api.request(f'tweets/:{CONVERSATION_ID}', { - 'tweet.fields':'author_id,conversation_id,created_at,in_reply_to_user_id' + 'tweet.fields':'author_id,conversation_id,created_at,referenced_tweets' }) for item in r: @@ -54,7 +61,7 @@ def print_tree(self, level): pager = TwitterPager(api, 'tweets/search/recent', { 'query':f'conversation_id:{CONVERSATION_ID}', - 'tweet.fields':'author_id,conversation_id,created_at,in_reply_to_user_id' + 'tweet.fields':'author_id,conversation_id,created_at,referenced_tweets' }) # "wait=2" means wait 2 seconds between each request. @@ -66,9 +73,9 @@ def print_tree(self, level): for item in pager.get_iterator(wait=2): node = TreeNode(item) print(f'{node.id()} => {node.reply_to()}') - # COLLECT ANY ORPHANS THAT ARE NODE'S CHILD + # COLLECT ANY ORPHANS THAT ARE CHILDREN OF THE NEW NODE orphans = [orphan for orphan in orphans if not node.find_parent_of(orphan)] - # IF NODE CANNOT BE PLACED IN TREE, ORPHAN IT UNTIL ITS PARENT IS FOUND + # IF THE NEW NODE CANNOT BE PLACED IN TREE, ORPHAN IT UNTIL ITS PARENT IS FOUND if not root.find_parent_of(node): orphans.append(node) diff --git a/examples/v2/search_tweets.py b/examples/v2/search_tweets.py index 11e58f8..9928615 100644 --- a/examples/v2/search_tweets.py +++ b/examples/v2/search_tweets.py @@ -5,11 +5,18 @@ try: o = TwitterOAuth.read_file() api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') - r = api.request('tweets/search/recent', {'query':QUERY, 'tweet.fields':'public_metrics'}) + r = api.request('tweets/search/recent', { + 'query':QUERY, + 'tweet.fields':'author_id', + 'expansions':'author_id'}) for item in r: print(item) + print('\nINCLUDES') + print(r.json()['includes']) + + print('\nQUOTA') print(r.get_quota()) except TwitterRequestError as e: From af06978c4d935579810e61de895b6e67a515d7eb Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Wed, 27 Jan 2021 16:48:37 -0500 Subject: [PATCH 067/139] fixed bug --- TwitterAPI/__init__.py | 2 +- TwitterAPI/constants.py | 4 ++-- examples/v2/conversation_tree.py | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index f682c81..3c5a35b 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.4' +__version__ = '2.6.4.1' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 198ea2b..693bafd 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -239,8 +239,8 @@ 'tweets': ('GET', 'api'), 'tweets/:PARAM/hidden': ('PUT', 'api'), # ID 'tweets/:PARAM': ('GET', 'api'), # ID - 'tweets/compliance/jobs' ('POST', 'api'), # use method_override for 'GET' - 'tweets/compliance/jobs/:PARAM' ('POST', 'api'), # ID + 'tweets/compliance/jobs': ('POST', 'api'), # use method_override for 'GET' + 'tweets/compliance/jobs/:PARAM': ('POST', 'api'), # ID 'tweets/sample/stream': ('GET', 'api'), 'tweets/search/all': ('GET', 'api'), 'tweets/search/recent': ('GET', 'api'), diff --git a/examples/v2/conversation_tree.py b/examples/v2/conversation_tree.py index 8cec7dc..abe8ed2 100644 --- a/examples/v2/conversation_tree.py +++ b/examples/v2/conversation_tree.py @@ -8,24 +8,24 @@ def __init__(self, data): """data is a tweet's json object""" self.data = data self.children = [] - self.parent = None + self.replied_to_tweet = None if 'referenced_tweets' in self.data: for tweet in self.data['referenced_tweets']: if tweet['type'] == 'replied_to': - self.parent = tweet['id'] + self.replied_to_tweet = tweet['id'] break def id(self): """a node is identified by its tweet id""" return self.data['id'] - def reply_to(self): + def parent(self): """the reply-to tweet is the parent of the node""" - return self.parent + return self.replied_to_tweet def find_parent_of(self, node): """append a node to the children of it's parent tweet""" - if node.reply_to() == self.id(): + if node.parent() == self.id(): self.children.append(node) return True for child in self.children: @@ -72,7 +72,7 @@ def print_tree(self, level): for item in pager.get_iterator(wait=2): node = TreeNode(item) - print(f'{node.id()} => {node.reply_to()}') + print(f'{node.id()} => {node.parent()}') # COLLECT ANY ORPHANS THAT ARE CHILDREN OF THE NEW NODE orphans = [orphan for orphan in orphans if not node.find_parent_of(orphan)] # IF THE NEW NODE CANNOT BE PLACED IN TREE, ORPHAN IT UNTIL ITS PARENT IS FOUND From 9d298adb971054734d9a9f9cb219b341a58c79a4 Mon Sep 17 00:00:00 2001 From: Dylan Caponi Date: Wed, 27 Jan 2021 19:07:31 -0500 Subject: [PATCH 068/139] Add promoted_tweets --- TwitterAPI/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 693bafd..7a36478 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -194,6 +194,7 @@ # ADS API (not tested!!) 'accounts/:PARAM/auction_insights': ('GET', 'ads-api'), # ACCOUNT ID + 'accounts/:PARAM/promoted_tweets': ('GET', 'ads-api'), # ACCOUNT ID 'stats/accounts/:PARAM': ('GET', 'ads-api'), # ACCOUNT ID 'stats/accounts/:PARAM/reach/funding_instruments': ('GET', 'ads-api'), # ACCOUNT ID 'stats/jobs/accounts/:PARAM': ('GET', 'ads-api'), # ACCOUNT ID From 7568b860099ac7fa4744a2cf6556aad6aed1a6ad Mon Sep 17 00:00:00 2001 From: Dylan Caponi Date: Thu, 28 Jan 2021 13:55:38 -0500 Subject: [PATCH 069/139] Update ads version --- TwitterAPI/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 7a36478..a6f73cc 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -19,7 +19,7 @@ VERSION = '1.1' CURATOR_VERSION = 'broadcast/1' -ADS_VERSION = '2' +ADS_VERSION = '8' ENDPOINTS = { # resource: (method, subdomain) From 65411da800c16bfdbe8bcc83e71f13d2f6520654 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sat, 30 Jan 2021 09:50:25 -0500 Subject: [PATCH 070/139] merged dylancaponi-patch-1 with Ads endpoints --- CHANGE.log | 2 ++ TwitterAPI/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGE.log b/CHANGE.log index 645ab76..026a921 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -156,3 +156,5 @@ v2.6.2, 27 Oct 2020 -- Merged ckovamees-master with add media endpoints. v2.6.3, 12 Jan 2021 -- Merged peguerosdc-master with v2 user endpoints. v2.6.4, 27 Jan 2021 -- Fixed conversation_tree.py example. + +v2.6.5, 30 Jan 2021 -- Merged dylancaponi-patch-1 with Ads endpoints. diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 3c5a35b..2f6ce99 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.4.1' +__version__ = '2.6.5' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From ad7fe50bf0ac17d44cd16d0be658edcc0b0e1c8f Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 15 Feb 2021 18:22:37 -0500 Subject: [PATCH 071/139] Fixed v2 next/prev token in TwitterPager --- CHANGE.log | 2 ++ TwitterAPI/TwitterPager.py | 6 ++++-- TwitterAPI/__init__.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index 026a921..6610787 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -158,3 +158,5 @@ v2.6.3, 12 Jan 2021 -- Merged peguerosdc-master with v2 user endpoints. v2.6.4, 27 Jan 2021 -- Fixed conversation_tree.py example. v2.6.5, 30 Jan 2021 -- Merged dylancaponi-patch-1 with Ads endpoints. + +v2.6.6, 14 Feb 2021 -- Fixed v2 next/prev token in TwitterPager. diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py index d1fa5bb..4af437d 100644 --- a/TwitterAPI/TwitterPager.py +++ b/TwitterAPI/TwitterPager.py @@ -23,6 +23,8 @@ class TwitterPager(object): def __init__(self, api, resource, params=None): self.api = api self.resource = resource + if not params: + params = {} self.params = params def get_iterator(self, wait=5, new_tweets=False): @@ -114,9 +116,9 @@ def get_iterator(self, wait=5, new_tweets=False): continue else: # VERSION 2 if new_tweets: - self.params['since_id'] = meta['newest_id'] + self.params['pagination_token'] = meta['previous_token'] else: - self.params['next_token'] = meta['next_token'] + self.params['pagination_token'] = meta['next_token'] except TwitterRequestError as e: if e.status_code < 500: diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 2f6ce99..5f01476 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.5' +__version__ = '2.6.6' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From c1f3e849ad5b0cd316f70cf17d28078f9d12b2bd Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 15 Feb 2021 18:23:52 -0500 Subject: [PATCH 072/139] Fixed v2 next/prev token in TwitterPager --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 226971a..e974823 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ MANIFEST +.DS_Store + *.py[cod] # C extensions @@ -36,4 +38,4 @@ nosetests.xml .project .pydevproject .idea/* -env/* \ No newline at end of file +env/* From 8035dc110106c31e99ca0d6d10704bf9b3fd9507 Mon Sep 17 00:00:00 2001 From: Patrick <24828335+factorlive@users.noreply.github.com> Date: Wed, 17 Feb 2021 23:32:39 +0000 Subject: [PATCH 073/139] Swap followers and friends for example Adjusting example, otherwise, ids remain in the set that cannot be unfollowed. --- examples/v1.1/unfollow_friends.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/v1.1/unfollow_friends.py b/examples/v1.1/unfollow_friends.py index abc1ef7..65a0cb5 100644 --- a/examples/v1.1/unfollow_friends.py +++ b/examples/v1.1/unfollow_friends.py @@ -7,12 +7,12 @@ , ) -followers = set(id for id in api.request('followers/ids')) friends = set(id for id in api.request('friends/ids')) -unfollow = set(followers) - set(friends) +followers = set(id for id in api.request('followers/ids')) +unfollow = set(friends) - set(followers) for id in unfollow: r = api.request('friendships/destroy', {'user_id': id}) if r.status_code == 200: status = r.json() - print 'unfollowed %s' % status['screen_name'] \ No newline at end of file + print 'unfollowed %s' % status['screen_name'] From 94669e0c20ef3362ecb9ad1b87cac2b1ddee364b Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 19 Feb 2021 16:50:51 -0500 Subject: [PATCH 074/139] Fixed v2 next/prev token in TwitterPager for search --- CHANGE.log | 2 ++ TwitterAPI/TwitterAPI.py | 2 +- TwitterAPI/TwitterPager.py | 8 +++++--- TwitterAPI/__init__.py | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index 6610787..82f3e6b 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -160,3 +160,5 @@ v2.6.4, 27 Jan 2021 -- Fixed conversation_tree.py example. v2.6.5, 30 Jan 2021 -- Merged dylancaponi-patch-1 with Ads endpoints. v2.6.6, 14 Feb 2021 -- Fixed v2 next/prev token in TwitterPager. + +v2.6.7, 19 Feb 2021 -- Fixed v2 next/prev token in TwitterPager for search. diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index e6dfa38..cbf0c05 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -372,4 +372,4 @@ def __iter__(self): yield json.loads(item.decode('utf8')) except ValueError as e: # invalid JSON string (possibly an unformatted error message) - raise TwitterConnectionError(e) + raise TwitterConnectionError(e) \ No newline at end of file diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py index 4af437d..9bee118 100644 --- a/TwitterAPI/TwitterPager.py +++ b/TwitterAPI/TwitterPager.py @@ -36,7 +36,6 @@ def get_iterator(self, wait=5, new_tweets=False): :param new_tweets: Boolean determining the search direction. False (default) retrieves old results. True retrieves current results. - :param version: override default API version or None (default) :returns: JSON objects containing statuses, errors or other return info. """ @@ -115,10 +114,13 @@ def get_iterator(self, wait=5, new_tweets=False): else: continue else: # VERSION 2 + # TWITTER SHOULD STANDARDIZE ON pagination_token IN THE FUTURE + SEARCH_ENDPOINTS = ['tweets/search/recent', 'tweets/search/all'] + pagination_token = 'next_token' if self.resource in SEARCH_ENDPOINTS else 'pagination_token' if new_tweets: - self.params['pagination_token'] = meta['previous_token'] + self.params[pagination_token] = meta['previous_token'] else: - self.params['pagination_token'] = meta['next_token'] + self.params[pagination_token] = meta['next_token'] except TwitterRequestError as e: if e.status_code < 500: diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 5f01476..48d4281 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.6' +__version__ = '2.6.7' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 21ef07d86ed39dcb94e7c011f6673dd40a7543be Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 19 Feb 2021 18:11:37 -0500 Subject: [PATCH 075/139] Added v2 tweet hydration --- CHANGE.log | 2 + TwitterAPI/TwitterAPI.py | 170 ++++++++++++++++++--------- TwitterAPI/TwitterPager.py | 9 +- TwitterAPI/__init__.py | 2 +- examples/v2/conversation_tree.py | 41 +++---- examples/v2/lookup_tweet.py | 3 +- examples/v2/lookup_tweet_hydrate.py | 35 ++++++ examples/v2/page_tweets.py | 1 + examples/v2/page_tweets_hydrate.py | 36 ++++++ examples/v2/search_tweets_hydrate.py | 37 ++++++ examples/v2/stream_tweets_hydrate.py | 50 ++++++++ 11 files changed, 301 insertions(+), 85 deletions(-) create mode 100644 examples/v2/lookup_tweet_hydrate.py create mode 100644 examples/v2/page_tweets_hydrate.py create mode 100644 examples/v2/search_tweets_hydrate.py create mode 100644 examples/v2/stream_tweets_hydrate.py diff --git a/CHANGE.log b/CHANGE.log index 82f3e6b..87a8bd1 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -162,3 +162,5 @@ v2.6.5, 30 Jan 2021 -- Merged dylancaponi-patch-1 with Ads endpoints. v2.6.6, 14 Feb 2021 -- Fixed v2 next/prev token in TwitterPager. v2.6.7, 19 Feb 2021 -- Fixed v2 next/prev token in TwitterPager for search. + +v2.6.8, 19 Feb 2021 -- Added v2 tweet hydration. diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index cbf0c05..bce2c80 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -11,11 +11,11 @@ from requests_oauthlib import OAuth1 from .TwitterError import * import json +import os import requests import socket import ssl import time -import os DEFAULT_USER_AGENT = os.getenv('DEFAULT_USER_AGENT', 'python-TwitterAPI') @@ -85,7 +85,7 @@ def __init__( else: raise Exception('Unknown oAuth version') - def _prepare_url(self, subdomain, path, api_version): + def _prepare_url(self, subdomain, path): if subdomain == 'curator': return '%s://%s.%s/%s/%s.json' % (PROTOCOL, subdomain, @@ -103,17 +103,17 @@ def _prepare_url(self, subdomain, path, api_version): subdomain, DOMAIN, path) - elif api_version == '1.1': + elif self.version == '1.1': return '%s://%s.%s/%s/%s.json' % (PROTOCOL, subdomain, DOMAIN, - api_version, + self.version, path) - elif api_version == '2': + elif self.version == '2': return '%s://%s.%s/%s/%s' % (PROTOCOL, subdomain, DOMAIN, - api_version, + self.version, path) else: raise Exception('Unsupported API version') @@ -130,14 +130,17 @@ def _get_endpoint(self, resource): else: return (resource, resource) - def request(self, resource, params=None, files=None, method_override=None, api_version=None): + def request(self, resource, params=None, files=None, method_override=None, hydrate_tweets=False): """Request a Twitter REST API or Streaming API resource. :param resource: A valid Twitter endpoint (ex. "search/tweets") :param params: Dictionary with endpoint parameters or None (default) :param files: Dictionary with multipart-encoded file or None (default) :param method_override: Request method to override or None (default) - :param api_version: override default API version or None (default) + :param hydrate_tweets: Boolean determining whether to insert expansion data from + "includes" directly into the tweet "data" structure. + False (default) does not hydrate. + True hydrates. :returns: TwitterResponse :raises: TwitterConnectionError @@ -151,10 +154,8 @@ def request(self, resource, params=None, files=None, method_override=None, api_v method, subdomain = ENDPOINTS[endpoint] if method_override: method = method_override - if not api_version: - api_version = self.version - url = self._prepare_url(subdomain, resource, api_version) - if api_version == '1.1' and 'stream' in subdomain: + url = self._prepare_url(subdomain, resource) + if self.version == '1.1' and 'stream' in subdomain: session.stream = True timeout = self.STREAMING_TIMEOUT # always use 'delimited' for efficient stream parsing @@ -162,7 +163,7 @@ def request(self, resource, params=None, files=None, method_override=None, api_v params = {} params['delimited'] = 'length' params['stall_warning'] = 'true' - elif api_version == '2' and resource.endswith('/stream'): + elif self.version == '2' and resource.endswith('/stream'): session.stream = True timeout = self.STREAMING_TIMEOUT else: @@ -170,7 +171,7 @@ def request(self, resource, params=None, files=None, method_override=None, api_v timeout = self.REST_TIMEOUT d = p = j = None if method == 'POST': - if api_version == '1.1': + if self.version == '1.1': d = params else: j = params @@ -178,20 +179,35 @@ def request(self, resource, params=None, files=None, method_override=None, api_v j = params else: p = params + try: - r = session.request( - method, - url, - data=d, - params=p, - json=j, - timeout=(self.CONNECTION_TIMEOUT, timeout), - files=files, - proxies=self.proxies) + if False and method == 'PUT': + session.headers['Content-type'] = 'application/json' + data = params + r = session.request( + method, + url, + json=data) + else: + r = session.request( + method, + url, + data=d, + params=p, + json=j, + timeout=(self.CONNECTION_TIMEOUT, timeout), + files=files, + proxies=self.proxies) except (ConnectionError, ProtocolError, ReadTimeout, ReadTimeoutError, SSLError, ssl.SSLError, socket.error) as e: raise TwitterConnectionError(e) - return TwitterResponse(r, session.stream) + + options = { + 'api_version': self.version, + 'is_stream': session.stream, + 'hydrate_tweets': hydrate_tweets + } + return TwitterResponse(r, options) class TwitterResponse(object): @@ -200,11 +216,12 @@ class TwitterResponse(object): :param response: The requests.Response object returned by the API call :param stream: Boolean connection type (True if a streaming connection) + :param options: Dict containing parsing options """ - def __init__(self, response, stream): + def __init__(self, response, options): self.response = response - self.stream = stream + self.options = options @property def headers(self): @@ -239,10 +256,10 @@ def get_iterator(self): if self.response.status_code != 200: raise TwitterRequestError(self.response.status_code, msg=self.response.text) - if self.stream: - return iter(_StreamingIterable(self.response)) + if self.options['is_stream']: + return iter(_StreamingIterable(self.response, self.options)) else: - return iter(_RestIterable(self.response)) + return iter(_RestIterable(self.response, self.options)) def __iter__(self): """Get API dependent iterator. @@ -278,33 +295,47 @@ class _RestIterable(object): """Iterate statuses, errors or other iterable objects in a REST API response. :param response: The request.Response from a Twitter REST API request + :param options: Dict containing parsing options """ - def __init__(self, response): + def __init__(self, response, options): resp = response.json() - # convert json response into something iterable - if 'errors' in resp: - self.results = resp['errors'] - elif 'statuses' in resp: - self.results = resp['statuses'] - elif 'users' in resp: - self.results = resp['users'] - elif 'ids' in resp: - self.results = resp['ids'] - elif 'results' in resp: - self.results = resp['results'] - elif 'data' in resp: - if not isinstance(resp['data'], dict): - self.results = resp['data'] + if options['api_version'] == '2': + if 'data' in resp: + if not isinstance(resp['data'], dict): + self.results = resp['data'] + else: + self.results = [resp['data']] else: - self.results = [resp['data']] - elif hasattr(resp, '__iter__') and not isinstance(resp, dict): - if len(resp) > 0 and 'trends' in resp[0]: - self.results = resp[0]['trends'] + self.results = [] + if 'includes' in resp and options['hydrate_tweets']: + self.results = _hydrate_tweets(self.results, resp['includes']) + elif options['api_version'] == '1.1': + # convert json response into something iterable + if 'errors' in resp: + self.results = resp['errors'] + elif 'statuses' in resp: + self.results = resp['statuses'] + elif 'users' in resp: + self.results = resp['users'] + elif 'ids' in resp: + self.results = resp['ids'] + elif 'results' in resp: + self.results = resp['results'] + elif 'data' in resp: + if not isinstance(resp['data'], dict): + self.results = resp['data'] + else: + self.results = [resp['data']] + elif hasattr(resp, '__iter__') and not isinstance(resp, dict): + if len(resp) > 0 and 'trends' in resp[0]: + self.results = resp[0]['trends'] + else: + self.results = resp else: - self.results = resp + self.results = (resp,) else: - self.results = (resp,) + self.results = [] def __iter__(self): """Return a tweet status as a JSON object.""" @@ -317,10 +348,12 @@ class _StreamingIterable(object): """Iterate statuses or other objects in a Streaming API response. :param response: The request.Response from a Twitter Streaming API request + :param options: Dict containing parsing options """ - def __init__(self, response): + def __init__(self, response, options): self.stream = response.raw + self.options = options def _iter_stream(self): """Stream parser. @@ -352,9 +385,14 @@ def _iter_stream(self): item = None item = self.stream.read(nbytes) break + item = json.loads(item.decode('utf8')) + if self.options['api_version'] == '2': + if self.options['hydrate_tweets']: + if 'data' in item and 'includes' in item: + item = _hydrate_tweets(item['data'], item['includes']) yield item except (ConnectionError, ProtocolError, ReadTimeout, ReadTimeoutError, - SSLError, ssl.SSLError, socket.error) as e: + SSLError, ssl.SSLError, socket.error, ValueError) as e: raise TwitterConnectionError(e) except AttributeError: # inform iterator to exit when client closes connection @@ -368,8 +406,26 @@ def __iter__(self): """ for item in self._iter_stream(): if item: - try: - yield json.loads(item.decode('utf8')) - except ValueError as e: - # invalid JSON string (possibly an unformatted error message) - raise TwitterConnectionError(e) \ No newline at end of file + yield item + + +def _hydrate_tweets(data, includes): + """Insert expansion fields back into tweet data. + + :returns: Tweet status as a JSON object. + """ + substitutes = [] + for key in includes: + incl = includes[key] + for obj in incl: + if 'id' in obj: + replace = ('id', obj['id'], obj) + elif 'media_key' in obj: + replace = ('media_key', obj['media_key'], obj) + else: + raise Exception('Unsupported "includes" object: %s' % key) + substitutes.append(replace) + data = json.dumps(data) + for replace in substitutes: + data = data.replace('"' + replace[1] + '"', json.dumps(replace[2])) + return json.loads(data) \ No newline at end of file diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py index 9bee118..6ee08a3 100644 --- a/TwitterAPI/TwitterPager.py +++ b/TwitterAPI/TwitterPager.py @@ -18,14 +18,19 @@ class TwitterPager(object): :param api: An authenticated TwitterAPI object :param resource: String with the resource path (ex. search/tweets) :param params: Dictionary of resource parameters + :param hydrate_tweets: Boolean determining whether to insert expansion data from + "includes" directly into the tweet "data" structure. + False (default) does not hydrate. + True hydrates. """ - def __init__(self, api, resource, params=None): + def __init__(self, api, resource, params=None, hydrate_tweets=False): self.api = api self.resource = resource if not params: params = {} self.params = params + self.hydrate_tweets = hydrate_tweets def get_iterator(self, wait=5, new_tweets=False): """Iterate response from Twitter REST API resource. Resource is called @@ -44,7 +49,7 @@ def get_iterator(self, wait=5, new_tweets=False): try: # REQUEST ONE PAGE OF RESULTS... start = time.time() - r = self.api.request(self.resource, self.params) + r = self.api.request(self.resource, self.params, hydrate_tweets=self.hydrate_tweets) it = r.get_iterator() if new_tweets: it = reversed(list(it)) diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 48d4281..1f9d5b3 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.7' +__version__ = '2.6.8' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/examples/v2/conversation_tree.py b/examples/v2/conversation_tree.py index abe8ed2..82f5a9c 100644 --- a/examples/v2/conversation_tree.py +++ b/examples/v2/conversation_tree.py @@ -1,31 +1,25 @@ from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager -from datetime import datetime -CONVERSATION_ID = '1354097088179687425' +# NOTE: If conversation is over a week old then it will not get returned. +CONVERSATION_ID = '1362514443566014471' class TreeNode: def __init__(self, data): """data is a tweet's json object""" self.data = data self.children = [] - self.replied_to_tweet = None - if 'referenced_tweets' in self.data: - for tweet in self.data['referenced_tweets']: - if tweet['type'] == 'replied_to': - self.replied_to_tweet = tweet['id'] - break def id(self): - """a node is identified by its tweet id""" - return self.data['id'] + """a node is identified by its author""" + return self.data['author_id'] - def parent(self): - """the reply-to tweet is the parent of the node""" - return self.replied_to_tweet + def reply_to(self): + """the reply-to user is the parent of the node""" + return self.data['in_reply_to_user_id'] def find_parent_of(self, node): - """append a node to the children of it's parent tweet""" - if node.parent() == self.id(): + """append a node to the children of it's reply-to user""" + if node.reply_to() == self.id(): self.children.append(node) return True for child in self.children: @@ -35,9 +29,9 @@ def find_parent_of(self, node): def print_tree(self, level): """level 0 is the root node, then incremented for subsequent generations""" - print(f'{level*"_"}{level}: [{self.data["created_at"]}] {self.data["text"][0:40]}') + print(f'{level*"_"}{level}: {self.id()}') level += 1 - for child in reversed(self.children): + for child in self.children: child.print_tree(level) try: @@ -48,7 +42,7 @@ def print_tree(self, level): r = api.request(f'tweets/:{CONVERSATION_ID}', { - 'tweet.fields':'author_id,conversation_id,created_at,referenced_tweets' + 'tweet.fields': 'author_id,conversation_id,created_at,in_reply_to_user_id' }) for item in r: @@ -60,8 +54,8 @@ def print_tree(self, level): pager = TwitterPager(api, 'tweets/search/recent', { - 'query':f'conversation_id:{CONVERSATION_ID}', - 'tweet.fields':'author_id,conversation_id,created_at,referenced_tweets' + 'query': f'conversation_id:{CONVERSATION_ID}', + 'tweet.fields': 'author_id,conversation_id,created_at,in_reply_to_user_id' }) # "wait=2" means wait 2 seconds between each request. @@ -70,12 +64,13 @@ def print_tree(self, level): orphans = [] + print('\nBUILDING TREE...') for item in pager.get_iterator(wait=2): node = TreeNode(item) - print(f'{node.id()} => {node.parent()}') - # COLLECT ANY ORPHANS THAT ARE CHILDREN OF THE NEW NODE + print(f'{node.id()} => {node.reply_to()}') + # COLLECT ANY ORPHANS THAT ARE NODE'S CHILD orphans = [orphan for orphan in orphans if not node.find_parent_of(orphan)] - # IF THE NEW NODE CANNOT BE PLACED IN TREE, ORPHAN IT UNTIL ITS PARENT IS FOUND + # IF NODE CANNOT BE PLACED IN TREE, ORPHAN IT UNTIL ITS PARENT IS FOUND if not root.find_parent_of(node): orphans.append(node) diff --git a/examples/v2/lookup_tweet.py b/examples/v2/lookup_tweet.py index a8ae641..c1e2dc6 100644 --- a/examples/v2/lookup_tweet.py +++ b/examples/v2/lookup_tweet.py @@ -2,7 +2,6 @@ TWEET_ID = '964575983633252353' - try: o = TwitterOAuth.read_file() api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') @@ -22,4 +21,4 @@ print(e) except Exception as e: - print(e) + print(e) \ No newline at end of file diff --git a/examples/v2/lookup_tweet_hydrate.py b/examples/v2/lookup_tweet_hydrate.py new file mode 100644 index 0000000..dc05424 --- /dev/null +++ b/examples/v2/lookup_tweet_hydrate.py @@ -0,0 +1,35 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +TWEET_ID = '1360029234773770240' + +EXPANSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,in_reply_to_user_id,attachments.media_keys,attachments.poll_ids,geo.place_id,entities.mentions.username' +MEDIA_FIELDS = 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics' +TWEET_FIELDS = 'created_at,author_id,public_metrics' +USER_FIELDS = 'location,profile_image_url,verified' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets/:{TWEET_ID}', + { + 'expansions': EXPANSIONS, + 'tweet.fields': TWEET_FIELDS, + 'user.fields': USER_FIELDS, + 'media.fields': MEDIA_FIELDS, + }, + hydrate_tweets=True) + + for item in r: + print(json.dumps(item, indent=2)) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/page_tweets.py b/examples/v2/page_tweets.py index f61e88e..1dfb618 100644 --- a/examples/v2/page_tweets.py +++ b/examples/v2/page_tweets.py @@ -6,6 +6,7 @@ o = TwitterOAuth.read_file() api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') pager = TwitterPager(api, 'tweets/search/recent', {'query':QUERY}) + for item in pager.get_iterator(new_tweets=False): print(item) diff --git a/examples/v2/page_tweets_hydrate.py b/examples/v2/page_tweets_hydrate.py new file mode 100644 index 0000000..40c0448 --- /dev/null +++ b/examples/v2/page_tweets_hydrate.py @@ -0,0 +1,36 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager +import json + +QUERY = 'pizza' + +EXPANSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,in_reply_to_user_id,attachments.media_keys,attachments.poll_ids,geo.place_id,entities.mentions.username' +MEDIA_FIELDS = 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics' +TWEET_FIELDS = 'created_at,author_id,public_metrics' +USER_FIELDS = 'location,profile_image_url,verified' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + pager = TwitterPager(api, 'tweets/search/recent', + { + 'query': {QUERY}, + 'expansions': EXPANSIONS, + 'tweet.fields': TWEET_FIELDS, + 'user.fields': USER_FIELDS, + 'media.fields': MEDIA_FIELDS, + }, + hydrate_tweets=True) + + for item in pager.get_iterator(new_tweets=False): + print(json.dumps(item, indent=2)) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/search_tweets_hydrate.py b/examples/v2/search_tweets_hydrate.py new file mode 100644 index 0000000..fd6389d --- /dev/null +++ b/examples/v2/search_tweets_hydrate.py @@ -0,0 +1,37 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +QUERY = 'kitten' +EXPANSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,in_reply_to_user_id,attachments.media_keys' +MEDIA_FIELDS = 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics' +TWEET_FIELDS = 'created_at,author_id,public_metrics' +USER_FIELDS = 'location,profile_image_url,verified' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request('tweets/search/recent', + { + 'query': {QUERY}, + 'expansions': EXPANSIONS, + 'media.fields': MEDIA_FIELDS, + 'tweet.fields': TWEET_FIELDS, + 'user.fields': USER_FIELDS, + }, + hydrate_tweets=True) + + for item in r: + print(json.dumps(item, indent=2)) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/stream_tweets_hydrate.py b/examples/v2/stream_tweets_hydrate.py new file mode 100644 index 0000000..d8c5750 --- /dev/null +++ b/examples/v2/stream_tweets_hydrate.py @@ -0,0 +1,50 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +QUERY = '"pizza" OR "burger"' +EXPANSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,in_reply_to_user_id,attachments.media_keys,attachments.poll_ids,geo.place_id,entities.mentions.username' +TWEET_FIELDS='author_id,conversation_id,created_at,entities,geo,id,lang,public_metrics,source,text' +USER_FIELDS='created_at,description,entities,location,name,profile_image_url,public_metrics,url,username' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + + # ADD STREAM RULES + + r = api.request('tweets/search/stream/rules', {'add': [{'value':QUERY}]}) + print(f'[{r.status_code}] RULE ADDED: {json.dumps(r.json(), indent=2)}\n') + if r.status_code != 201: exit() + + # GET STREAM RULES + + r = api.request('tweets/search/stream/rules', method_override='GET') + print(f'[{r.status_code}] RULES: {json.dumps(r.json(), indent=2)}\n') + if r.status_code != 200: exit() + + # START STREAM + + r = api.request('tweets/search/stream', { + 'expansions': EXPANSIONS, + 'tweet.fields': TWEET_FIELDS, + 'user.fields': USER_FIELDS, + }, + hydrate_tweets=True) + print(f'[{r.status_code}] START...') + if r.status_code != 200: exit() + for item in r: + print(json.dumps(item, indent=2)) + +except KeyboardInterrupt: + print('\nDone!') + +except TwitterRequestError as e: + print(f'\n{e.status_code}') + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) From 67f3d8a80337e2f25851ee47526e1dd36c7da3a1 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sun, 21 Feb 2021 11:20:00 -0500 Subject: [PATCH 076/139] Fixed bug in _StreamingIterable --- CHANGE.log | 2 ++ TwitterAPI/TwitterAPI.py | 24 ++++++++++++++---------- TwitterAPI/__init__.py | 2 +- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index 87a8bd1..19e074b 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -164,3 +164,5 @@ v2.6.6, 14 Feb 2021 -- Fixed v2 next/prev token in TwitterPager. v2.6.7, 19 Feb 2021 -- Fixed v2 next/prev token in TwitterPager for search. v2.6.8, 19 Feb 2021 -- Added v2 tweet hydration. + +v2.6.9, 21 Feb 2021 -- Fixed bug in _StreamingIterable. \ No newline at end of file diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index bce2c80..24c4f8b 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -302,14 +302,14 @@ def __init__(self, response, options): resp = response.json() if options['api_version'] == '2': if 'data' in resp: - if not isinstance(resp['data'], dict): - self.results = resp['data'] + if isinstance(resp['data'], dict): + resp['data'] = [resp['data']] + if 'includes' in resp and options['hydrate_tweets']: + self.results = _hydrate_tweets(resp['data'], resp['includes']) else: - self.results = [resp['data']] + self.results = resp['data'] else: self.results = [] - if 'includes' in resp and options['hydrate_tweets']: - self.results = _hydrate_tweets(self.results, resp['includes']) elif options['api_version'] == '1.1': # convert json response into something iterable if 'errors' in resp: @@ -338,7 +338,9 @@ def __init__(self, response, options): self.results = [] def __iter__(self): - """Return a tweet status as a JSON object.""" + """ + :returns: Tweet status as a JSON object. + """ for item in self.results: yield item @@ -385,22 +387,22 @@ def _iter_stream(self): item = None item = self.stream.read(nbytes) break - item = json.loads(item.decode('utf8')) + if item: + item = json.loads(item.decode('utf8')) if self.options['api_version'] == '2': if self.options['hydrate_tweets']: if 'data' in item and 'includes' in item: item = _hydrate_tweets(item['data'], item['includes']) yield item except (ConnectionError, ProtocolError, ReadTimeout, ReadTimeoutError, - SSLError, ssl.SSLError, socket.error, ValueError) as e: + SSLError, ssl.SSLError, socket.error, ValueError, json.decoder.JSONDecodeError) as e: raise TwitterConnectionError(e) except AttributeError: # inform iterator to exit when client closes connection raise StopIteration def __iter__(self): - """Iterator. - + """ :returns: Tweet status as a JSON object. :raises: TwitterConnectionError """ @@ -412,6 +414,8 @@ def __iter__(self): def _hydrate_tweets(data, includes): """Insert expansion fields back into tweet data. + :param data: "data" property value in JSON response + :param includes: "includes" property value in JSON response :returns: Tweet status as a JSON object. """ substitutes = [] diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 1f9d5b3..5aa8b1e 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.8' +__version__ = '2.6.9' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From ed564be526fe22302c2511a9ce57d34b010130a2 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sun, 21 Feb 2021 12:27:06 -0500 Subject: [PATCH 077/139] Added Twitter badges to README --- README.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 9932a36..b6ec154 100755 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ |LOGO| ============================ -|BADGE_VERSION| |BADGE_CHAT| +|BADGE_VERSION| |BADGE_CHAT| |BADGE_1.1| |BADGE_1.1_PREMIUM| |BADGE_2| |BADGE_ADS8| |BADGE_LABS| ============================ .. |LOGO| image:: https://raw.githubusercontent.com/geduldig/TwitterAPI/master/logo.png @@ -10,6 +10,17 @@ :alt: Join the chat at https://gitter.im/geduldig/TwitterAPI :target: https://gitter.im/geduldig/TwitterAPI?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge +.. |BADGE_2| image:: https://img.shields.io/endpoint?url=https%3A%2F%2Fblue-sea-697d.quartiers047.workers.dev%3A443%2Fhttps%2Ftwbadges.glitch.me%2Fbadges%2Fv2 + :target: https://developer.twitter.com/en/docs/twitter-api +.. |BADGE_LABS| image:: https://img.shields.io/endpoint?url=https%3A%2F%2Fblue-sea-697d.quartiers047.workers.dev%3A443%2Fhttps%2Ftwbadges.glitch.me%2Fbadges%2Flabs + :target: https://developer.twitter.com/en/docs/labs +.. |BADGE_ADS8| image:: https://img.shields.io/endpoint?url=https%3A%2F%2Fblue-sea-697d.quartiers047.workers.dev%3A443%2Fhttps%2Ftwbadges.glitch.me%2Fbadges%2Fadsv8 + :target: https://developer.twitter.com/en/docs/twitter-ads-api +.. |BADGE_1.1| image:: https://img.shields.io/endpoint?url=https%3A%2F%2Fblue-sea-697d.quartiers047.workers.dev%3A443%2Fhttps%2Ftwbadges.glitch.me%2Fbadges%2Fstandard + :target: https://developer.twitter.com/en/docs/twitter-api +.. |BADGE_PREMIUM| image:: https://img.shields.io/endpoint?url=https%3A%2F%2Fblue-sea-697d.quartiers047.workers.dev%3A443%2Fhttps%2Ftwbadges.glitch.me%2Fbadges%2Fpremium + :target: https://developer.twitter.com + TwitterAPI is a Python package for accessing Twitter's REST APIs and Streaming APIs. It supports both Version 1.1 and Version 2 endpoints. REST APIs that are supported are: Public API, Collections API, Curator API, Ads API, Webhook API, Premium Search API. From 78e5cb330fea0b57a3659aafa87b37b14ef1ccc0 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sun, 21 Feb 2021 12:28:10 -0500 Subject: [PATCH 078/139] Added Twitter badges to README --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index b6ec154..bd76f8b 100755 --- a/README.rst +++ b/README.rst @@ -1,6 +1,7 @@ |LOGO| ============================ -|BADGE_VERSION| |BADGE_CHAT| |BADGE_1.1| |BADGE_1.1_PREMIUM| |BADGE_2| |BADGE_ADS8| |BADGE_LABS| +|BADGE_VERSION| |BADGE_CHAT| +|BADGE_1.1| |BADGE_PREMIUM| |BADGE_2| |BADGE_ADS8| |BADGE_LABS| ============================ .. |LOGO| image:: https://raw.githubusercontent.com/geduldig/TwitterAPI/master/logo.png From 50a49d83629ee3967b62c03235767a9e80e0fcf8 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sun, 21 Feb 2021 12:28:46 -0500 Subject: [PATCH 079/139] Added Twitter badges to README --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index bd76f8b..b2ec926 100755 --- a/README.rst +++ b/README.rst @@ -1,7 +1,6 @@ |LOGO| ============================ -|BADGE_VERSION| |BADGE_CHAT| -|BADGE_1.1| |BADGE_PREMIUM| |BADGE_2| |BADGE_ADS8| |BADGE_LABS| +|BADGE_VERSION| |BADGE_CHAT| |BADGE_1.1| |BADGE_PREMIUM| |BADGE_2| |BADGE_ADS8| |BADGE_LABS| ============================ .. |LOGO| image:: https://raw.githubusercontent.com/geduldig/TwitterAPI/master/logo.png From 029588c028d213835d7c20dd2077914c99a49fe1 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sun, 21 Feb 2021 12:30:06 -0500 Subject: [PATCH 080/139] Added Twitter badges to README --- README.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/README.rst b/README.rst index b2ec926..645886c 100755 --- a/README.rst +++ b/README.rst @@ -1,5 +1,4 @@ |LOGO| -============================ |BADGE_VERSION| |BADGE_CHAT| |BADGE_1.1| |BADGE_PREMIUM| |BADGE_2| |BADGE_ADS8| |BADGE_LABS| ============================ From 463eabcc1c9814901ca67bd612e42ad52dade0e0 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sun, 21 Feb 2021 12:30:42 -0500 Subject: [PATCH 081/139] Added Twitter badges to README --- README.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 645886c..925068c 100755 --- a/README.rst +++ b/README.rst @@ -1,5 +1,8 @@ |LOGO| -|BADGE_VERSION| |BADGE_CHAT| |BADGE_1.1| |BADGE_PREMIUM| |BADGE_2| |BADGE_ADS8| |BADGE_LABS| +============================ +|BADGE_VERSION| |BADGE_CHAT| +============================ +|BADGE_1.1| |BADGE_PREMIUM| |BADGE_2| |BADGE_ADS8| |BADGE_LABS| ============================ .. |LOGO| image:: https://raw.githubusercontent.com/geduldig/TwitterAPI/master/logo.png From 497e1d2d1b3e6243ef56a87f3f59cc0b285c7184 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 15 Mar 2021 16:20:07 -0400 Subject: [PATCH 082/139] Updated docs, fixed conversation_tree example --- TwitterAPI/TwitterAPI.py | 2 +- TwitterAPI/__init__.py | 2 +- docs/README.rst | 11 ++++--- docs/conf.py | 9 ++++-- examples/v2/conversation_tree.py | 54 +++++++++++++++++++------------- 5 files changed, 47 insertions(+), 31 deletions(-) diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 24c4f8b..5554de6 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -33,7 +33,7 @@ class TwitterAPI(object): :param access_token_key: Twitter application access token key :param access_token_secret: Twitter application access token secret :param auth_type: "oAuth1" (default) or "oAuth2" - :param proxy_url: HTTPS proxy URL string (ex. "https://USER:PASSWORD@SERVER:PORT"), + :param proxy_url: HTTPS proxy URL string (ex. 'https://USER:PASSWORD@SERVER:PORT'), or dict of URLs (ex. {'http':'http://SERVER', 'https':'https://SERVER'}) """ diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 5aa8b1e..02a83fa 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.9' +__version__ = '2.6.9.1' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/docs/README.rst b/docs/README.rst index 9dc69ae..ab14108 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -3,22 +3,23 @@ Instructions for Generating Documentation with Sphinx MAKE NEW DOCS IN MASTER BRANCH:: - clone git@github.com:geduldig/TwitterAPI.git + git clone git@github.com:geduldig/TwitterAPI.git cd TwitterAPI/docs make html DOWNLOAD OLD DOCS FROM GH-PAGES BRANCH:: - cd TwitterAPI-docs - clone -b gh-pages git@github.com:geduldig/TwitterAPI.git + cd gh-pages + git clone -b gh-pages git@github.com:geduldig/TwitterAPI.git COPY NEW DOCS OVER OLD DOCS:: - cp TwitterAPI/docs/_build/html/*.* TwitterAPI-docs/TwitterAPI - cp -r TwitterAPI/docs/_build/html/* TwitterAPI-docs/TwitterAPI + cp TwitterAPI/docs/_build/html/*.* gh-pages/TwitterAPI + cp -r TwitterAPI/docs/_build/html/* gh-pages/TwitterAPI rm -r TwitterAPI/docs/_build UPLOAD NEW DOCS TO GH-PAGES BRANCH::: + cd gh-pages/TwitterAPI git commit -am "2.5 docs" git push origin gh-pages diff --git a/docs/conf.py b/docs/conf.py index 5f3d833..0da6b7a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,6 +30,9 @@ # ones. extensions = [ 'sphinx.ext.autodoc', + "sphinx.ext.intersphinx", + "sphinx.ext.mathjax", + "sphinx.ext.viewcode", ] # Add any paths that contain templates here, relative to this directory. @@ -46,16 +49,16 @@ # General information about the project. project = u'TwitterAPI' -copyright = u'2014, geduldig' +copyright = u'2021, geduldig' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '2.5' +version = '2.6' # The full version, including alpha/beta/rc tags. -release = '2.5' +release = '2.6' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/examples/v2/conversation_tree.py b/examples/v2/conversation_tree.py index 82f5a9c..588d79c 100644 --- a/examples/v2/conversation_tree.py +++ b/examples/v2/conversation_tree.py @@ -1,25 +1,31 @@ from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager -# NOTE: If conversation is over a week old then it will not get returned. -CONVERSATION_ID = '1362514443566014471' +# NOTE: If conversation is over a week old then it will not get returned. +CONVERSATION_ID = '1369393783482236933' class TreeNode: def __init__(self, data): """data is a tweet's json object""" self.data = data self.children = [] + self.replied_to_tweet = None + if 'referenced_tweets' in self.data: + for tweet in self.data['referenced_tweets']: + if tweet['type'] == 'replied_to': + self.replied_to_tweet = tweet['id'] + break def id(self): - """a node is identified by its author""" - return self.data['author_id'] + """a node is identified by its tweet id""" + return self.data['id'] - def reply_to(self): - """the reply-to user is the parent of the node""" - return self.data['in_reply_to_user_id'] + def parent(self): + """the reply-to tweet is the parent of the node""" + return self.replied_to_tweet def find_parent_of(self, node): - """append a node to the children of it's reply-to user""" - if node.reply_to() == self.id(): + """append a node to the children of it's parent tweet""" + if node.parent() == self.id(): self.children.append(node) return True for child in self.children: @@ -29,9 +35,12 @@ def find_parent_of(self, node): def print_tree(self, level): """level 0 is the root node, then incremented for subsequent generations""" - print(f'{level*"_"}{level}: {self.id()}') + created_at = self.data['created_at'] + username = self.data['author_id']['username'] + text_80chars = self.data['text'][0:80].replace('\n', ' ') + print(f'{level*"_"}{level}: [{created_at}][{username}] {text_80chars}') level += 1 - for child in self.children: + for child in reversed(self.children): child.print_tree(level) try: @@ -42,8 +51,10 @@ def print_tree(self, level): r = api.request(f'tweets/:{CONVERSATION_ID}', { - 'tweet.fields': 'author_id,conversation_id,created_at,in_reply_to_user_id' - }) + 'expansions':'author_id', + 'tweet.fields':'author_id,conversation_id,created_at,referenced_tweets' + }, + hydrate_tweets=True) for item in r: root = TreeNode(item) @@ -54,9 +65,11 @@ def print_tree(self, level): pager = TwitterPager(api, 'tweets/search/recent', { - 'query': f'conversation_id:{CONVERSATION_ID}', - 'tweet.fields': 'author_id,conversation_id,created_at,in_reply_to_user_id' - }) + 'query':f'conversation_id:{CONVERSATION_ID}', + 'expansions':'author_id', + 'tweet.fields':'author_id,conversation_id,created_at,referenced_tweets' + }, + hydrate_tweets=True) # "wait=2" means wait 2 seconds between each request. # The rate limit is 450 requests per 15 minutes, or @@ -64,13 +77,12 @@ def print_tree(self, level): orphans = [] - print('\nBUILDING TREE...') for item in pager.get_iterator(wait=2): node = TreeNode(item) - print(f'{node.id()} => {node.reply_to()}') - # COLLECT ANY ORPHANS THAT ARE NODE'S CHILD + print(f'{node.id()} => {node.parent()}', item['author_id']['username']) + # COLLECT ANY ORPHANS THAT ARE CHILDREN OF THE NEW NODE orphans = [orphan for orphan in orphans if not node.find_parent_of(orphan)] - # IF NODE CANNOT BE PLACED IN TREE, ORPHAN IT UNTIL ITS PARENT IS FOUND + # IF THE NEW NODE CANNOT BE PLACED IN TREE, ORPHAN IT UNTIL ITS PARENT IS FOUND if not root.find_parent_of(node): orphans.append(node) @@ -87,4 +99,4 @@ def print_tree(self, level): print(e) except Exception as e: - print(e) \ No newline at end of file + print(e) From 8e681b8e40aeb71c0b1e77f18052517a42c4a385 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Tue, 6 Apr 2021 17:37:21 -0400 Subject: [PATCH 083/139] Added 'mentions' to be hydrated --- CHANGE.log | 4 +++- TwitterAPI/TwitterAPI.py | 11 ++++------- TwitterAPI/__init__.py | 2 +- TwitterAPI/constants.py | 4 +++- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index 19e074b..11098a4 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -165,4 +165,6 @@ v2.6.7, 19 Feb 2021 -- Fixed v2 next/prev token in TwitterPager for search. v2.6.8, 19 Feb 2021 -- Added v2 tweet hydration. -v2.6.9, 21 Feb 2021 -- Fixed bug in _StreamingIterable. \ No newline at end of file +v2.6.9, 21 Feb 2021 -- Fixed bug in _StreamingIterable. + +v2.6.10, 06 Apr 2021 -- Added 'mentions' to be hydrated. diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 5554de6..71fcff9 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -422,13 +422,10 @@ def _hydrate_tweets(data, includes): for key in includes: incl = includes[key] for obj in incl: - if 'id' in obj: - replace = ('id', obj['id'], obj) - elif 'media_key' in obj: - replace = ('media_key', obj['media_key'], obj) - else: - raise Exception('Unsupported "includes" object: %s' % key) - substitutes.append(replace) + for field in ['id', 'media_key', 'username']: + if field in obj: + replace = (field, obj[field], obj) + substitutes.append(replace) data = json.dumps(data) for replace in substitutes: data = data.replace('"' + replace[1] + '"', json.dumps(replace[2])) diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 02a83fa..5ef38b4 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.9.1' +__version__ = '2.6.10' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index a6f73cc..0500cb2 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -250,7 +250,9 @@ 'users': ('GET', 'api'), 'users/:PARAM': ('GET', 'api'), # ID 'users/:PARAM/followers': ('GET', 'api'), # ID - 'users/:PARAM/following': ('GET', 'api'), # ID + 'users/:PARAM/following': ('GET', 'api'), # ID, use method_override for 'POST' + 'users/:PARAM/following/:PARAM': ('DELETE', 'api'), # USER SOURCE ID, USER TARGET ID + 'users/:PARAM/mentions': ('GET', 'api'), # ID 'users/:PARAM/tweets': ('GET', 'api'), # ID 'users/by': ('GET', 'api'), 'users/by/username/:PARAM': ('GET', 'api'), # USERNAME From d349ab5979159f1f16eec78151ae972a39793ed5 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 12 Apr 2021 16:14:09 -0400 Subject: [PATCH 084/139] Added APPEND and REPACE hydration --- CHANGE.log | 3 + TwitterAPI/TwitterAPI.py | 91 ++++++++++++++++++++-------- TwitterAPI/TwitterPager.py | 17 +++--- TwitterAPI/__init__.py | 4 +- TwitterAPI/constants.py | 2 + examples/v2/lookup_tweet_hydrate.py | 8 +-- examples/v2/page_tweets_hydrate.py | 4 +- examples/v2/search_tweets_hydrate.py | 4 +- examples/v2/stream_tweets_hydrate.py | 7 ++- 9 files changed, 93 insertions(+), 47 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index 11098a4..7c514cc 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -168,3 +168,6 @@ v2.6.8, 19 Feb 2021 -- Added v2 tweet hydration. v2.6.9, 21 Feb 2021 -- Fixed bug in _StreamingIterable. v2.6.10, 06 Apr 2021 -- Added 'mentions' to be hydrated. + +v2.7.0, 12 Apr 2021 -- Two types of hydrate: APPEND and REPLACE. + Two new enums: HydrateType, OAuthType. diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 71fcff9..916a675 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -5,11 +5,12 @@ from .BearerAuth import BearerAuth as OAuth2 from .constants import * +from .TwitterError import * from datetime import datetime +from enum import Enum from requests.exceptions import ConnectionError, ReadTimeout, SSLError from requests.packages.urllib3.exceptions import ReadTimeoutError, ProtocolError from requests_oauthlib import OAuth1 -from .TwitterError import * import json import os import requests @@ -24,6 +25,17 @@ DEFAULT_REST_TIMEOUT = os.getenv('DEFAULT_REST_TIMEOUT', 5) +class OAuthType(Enum): + OAUTH1 = 'oAuth1' + OAUTH2 = 'oAuth2' + + +class HydrateType(Enum): + NONE = 0 + APPEND = 1 + REPLACE = 2 + + class TwitterAPI(object): """Access REST API or Streaming API resources. @@ -49,7 +61,7 @@ def __init__( consumer_secret=None, access_token_key=None, access_token_secret=None, - auth_type='oAuth1', + auth_type=OAuthType.OAUTH1, proxy_url=None, api_version=VERSION): """Initialize with your Twitter application credentials""" @@ -66,7 +78,7 @@ def __init__( self.proxies = None # Twitter supports two types of authentication. - if auth_type == 'oAuth1': + if auth_type == OAuthType.OAUTH1: if not all([consumer_key, consumer_secret, access_token_key, access_token_secret]): raise Exception('Missing authentication parameter') self.auth = OAuth1( @@ -74,7 +86,7 @@ def __init__( consumer_secret, access_token_key, access_token_secret) - elif auth_type == 'oAuth2': + elif auth_type == OAuthType.OAUTH2: if not all([consumer_key, consumer_secret]): raise Exception('Missing authentication parameter') self.auth = OAuth2( @@ -130,17 +142,17 @@ def _get_endpoint(self, resource): else: return (resource, resource) - def request(self, resource, params=None, files=None, method_override=None, hydrate_tweets=False): + def request(self, resource, params=None, files=None, method_override=None, hydrate_type=HydrateType.NONE): """Request a Twitter REST API or Streaming API resource. - :param resource: A valid Twitter endpoint (ex. "search/tweets") + :param resource: A Twitter endpoint (ex. "search/tweets") :param params: Dictionary with endpoint parameters or None (default) :param files: Dictionary with multipart-encoded file or None (default) :param method_override: Request method to override or None (default) - :param hydrate_tweets: Boolean determining whether to insert expansion data from - "includes" directly into the tweet "data" structure. - False (default) does not hydrate. - True hydrates. + :param hydrate_type: HydrateType or int + Do not hydrate - NONE or 0 (default) + Append new field with '_hydrate' suffix with hydrate values - APPEND or 1 + Replace current field value with hydrate values - REPLACE or 2 :returns: TwitterResponse :raises: TwitterConnectionError @@ -205,7 +217,7 @@ def request(self, resource, params=None, files=None, method_override=None, hydra options = { 'api_version': self.version, 'is_stream': session.stream, - 'hydrate_tweets': hydrate_tweets + 'hydrate_type': hydrate_type } return TwitterResponse(r, options) @@ -304,8 +316,10 @@ def __init__(self, response, options): if 'data' in resp: if isinstance(resp['data'], dict): resp['data'] = [resp['data']] - if 'includes' in resp and options['hydrate_tweets']: - self.results = _hydrate_tweets(resp['data'], resp['includes']) + h_type = options['hydrate_type'] + if 'includes' in resp and h_type != HydrateType.NONE: + field_suffix = '' if h_type == HydrateType.REPLACE else '_hydrate' + self.results = _hydrate_tweets(resp['data'], resp['includes'], field_suffix) else: self.results = resp['data'] else: @@ -390,12 +404,14 @@ def _iter_stream(self): if item: item = json.loads(item.decode('utf8')) if self.options['api_version'] == '2': - if self.options['hydrate_tweets']: + h_type = self.options['hydrate_type'] + if h_type != HydrateType.NONE: if 'data' in item and 'includes' in item: - item = _hydrate_tweets(item['data'], item['includes']) + field_suffix = '' if h_type == HydrateType.REPLACE else '_hydrate' + item = _hydrate_tweets(item['data'], item['includes'], field_suffix) yield item - except (ConnectionError, ProtocolError, ReadTimeout, ReadTimeoutError, - SSLError, ssl.SSLError, socket.error, ValueError, json.decoder.JSONDecodeError) as e: + except (ConnectionError, ValueError, ProtocolError, ReadTimeout, ReadTimeoutError, + SSLError, ssl.SSLError, socket.error, json.decoder.JSONDecodeError) as e: raise TwitterConnectionError(e) except AttributeError: # inform iterator to exit when client closes connection @@ -411,22 +427,45 @@ def __iter__(self): yield item -def _hydrate_tweets(data, includes): - """Insert expansion fields back into tweet data. +def _hydrate_tweets(data, includes, field_suffix): + """Insert expansion fields back into tweet data by appending + a new field as a sibling to the referenced field. :param data: "data" property value in JSON response :param includes: "includes" property value in JSON response + :param field_suffix: Suffix appended to a hydrated field name. + Either "_hydrate" which puts hydrated values into + a new field, or "" which replaces the current + field value with hydrated values. + :returns: Tweet status as a JSON object. """ - substitutes = [] + new_fields = [] for key in includes: incl = includes[key] for obj in incl: for field in ['id', 'media_key', 'username']: if field in obj: - replace = (field, obj[field], obj) - substitutes.append(replace) - data = json.dumps(data) - for replace in substitutes: - data = data.replace('"' + replace[1] + '"', json.dumps(replace[2])) - return json.loads(data) \ No newline at end of file + _create_include_fields(data, (obj[field], obj), new_fields) + + for item in new_fields: + parent = item[0] + field = item[1] + field_suffix + include = item[2] + parent[field] = include + return data + + +def _create_include_fields(parent, include, new_fields): + """Depth-first seach into 'parent' to locate fields referenced in + 'include'. Each match is appended to 'new_fields'. + """ + if isinstance(parent, list): + for item in parent: + _create_include_fields(item, include, new_fields) + elif isinstance(parent, dict): + for key, value in parent.items(): + if value == include[0]: + new_fields.append((parent, key, include[1])) + else: + _create_include_fields(value, include, new_fields) \ No newline at end of file diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py index 6ee08a3..fb3ecd6 100644 --- a/TwitterAPI/TwitterPager.py +++ b/TwitterAPI/TwitterPager.py @@ -3,9 +3,10 @@ __license__ = "MIT" +from .TwitterAPI import HydrateType +from .TwitterError import * from requests.exceptions import ConnectionError, ReadTimeout, SSLError from requests.packages.urllib3.exceptions import ReadTimeoutError, ProtocolError -from .TwitterError import * import requests import time @@ -18,19 +19,19 @@ class TwitterPager(object): :param api: An authenticated TwitterAPI object :param resource: String with the resource path (ex. search/tweets) :param params: Dictionary of resource parameters - :param hydrate_tweets: Boolean determining whether to insert expansion data from - "includes" directly into the tweet "data" structure. - False (default) does not hydrate. - True hydrates. + :param hydrate_type: HydrateType or int + Do not hydrate - NONE or 0 (default) + Append new field with '_hydrate' suffix with hydrate values - APPEND or 1 + Replace current field value with hydrate values - REPLACE or 2 """ - def __init__(self, api, resource, params=None, hydrate_tweets=False): + def __init__(self, api, resource, params=None, hydrate_type=HydrateType.NONE): self.api = api self.resource = resource if not params: params = {} self.params = params - self.hydrate_tweets = hydrate_tweets + self.hydrate_type = hydrate_type def get_iterator(self, wait=5, new_tweets=False): """Iterate response from Twitter REST API resource. Resource is called @@ -49,7 +50,7 @@ def get_iterator(self, wait=5, new_tweets=False): try: # REQUEST ONE PAGE OF RESULTS... start = time.time() - r = self.api.request(self.resource, self.params, hydrate_tweets=self.hydrate_tweets) + r = self.api.request(self.resource, self.params, hydrate_type=self.hydrate_type) it = r.get_iterator() if new_tweets: it = reversed(list(it)) diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 5ef38b4..bbc8399 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.6.10' +__version__ = '2.7.0' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' @@ -13,7 +13,7 @@ try: - from .TwitterAPI import TwitterAPI, TwitterResponse + from .TwitterAPI import TwitterAPI, TwitterResponse, OAuthType, HydrateType from .TwitterError import TwitterConnectionError, TwitterRequestError from .TwitterOAuth import TwitterOAuth from .TwitterPager import TwitterPager diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 0500cb2..efc3930 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -17,10 +17,12 @@ PROTOCOL = 'https' DOMAIN = 'twitter.com' + VERSION = '1.1' CURATOR_VERSION = 'broadcast/1' ADS_VERSION = '8' + ENDPOINTS = { # resource: (method, subdomain) diff --git a/examples/v2/lookup_tweet_hydrate.py b/examples/v2/lookup_tweet_hydrate.py index dc05424..00bde04 100644 --- a/examples/v2/lookup_tweet_hydrate.py +++ b/examples/v2/lookup_tweet_hydrate.py @@ -1,11 +1,11 @@ -from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, HydrateType import json TWEET_ID = '1360029234773770240' EXPANSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,in_reply_to_user_id,attachments.media_keys,attachments.poll_ids,geo.place_id,entities.mentions.username' MEDIA_FIELDS = 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics' -TWEET_FIELDS = 'created_at,author_id,public_metrics' +TWEET_FIELDS = 'created_at,author_id,public_metrics,context_annotations,entities' USER_FIELDS = 'location,profile_image_url,verified' try: @@ -18,10 +18,10 @@ 'user.fields': USER_FIELDS, 'media.fields': MEDIA_FIELDS, }, - hydrate_tweets=True) + hydrate_type=HydrateType.APPEND) for item in r: - print(json.dumps(item, indent=2)) + print(json.dumps(item, indent=2, sort_keys=True)) except TwitterRequestError as e: print(e.status_code) diff --git a/examples/v2/page_tweets_hydrate.py b/examples/v2/page_tweets_hydrate.py index 40c0448..e32117c 100644 --- a/examples/v2/page_tweets_hydrate.py +++ b/examples/v2/page_tweets_hydrate.py @@ -1,4 +1,4 @@ -from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager, HydrateType import json QUERY = 'pizza' @@ -19,7 +19,7 @@ 'user.fields': USER_FIELDS, 'media.fields': MEDIA_FIELDS, }, - hydrate_tweets=True) + hydrate_type=HydrateType.APPEND) for item in pager.get_iterator(new_tweets=False): print(json.dumps(item, indent=2)) diff --git a/examples/v2/search_tweets_hydrate.py b/examples/v2/search_tweets_hydrate.py index fd6389d..0704a99 100644 --- a/examples/v2/search_tweets_hydrate.py +++ b/examples/v2/search_tweets_hydrate.py @@ -1,4 +1,4 @@ -from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, HydrateType import json QUERY = 'kitten' @@ -18,7 +18,7 @@ 'tweet.fields': TWEET_FIELDS, 'user.fields': USER_FIELDS, }, - hydrate_tweets=True) + hydrate_type=HydrateType.APPEND) for item in r: print(json.dumps(item, indent=2)) diff --git a/examples/v2/stream_tweets_hydrate.py b/examples/v2/stream_tweets_hydrate.py index d8c5750..4fb93d5 100644 --- a/examples/v2/stream_tweets_hydrate.py +++ b/examples/v2/stream_tweets_hydrate.py @@ -1,4 +1,4 @@ -from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, HydrateType, OAuthType import json QUERY = '"pizza" OR "burger"' @@ -8,7 +8,7 @@ try: o = TwitterOAuth.read_file() - api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type=OAuthType.OAUTH2, api_version='2') # ADD STREAM RULES @@ -29,7 +29,8 @@ 'tweet.fields': TWEET_FIELDS, 'user.fields': USER_FIELDS, }, - hydrate_tweets=True) + hydrate_type=HydrateType.APPEND) + print(f'[{r.status_code}] START...') if r.status_code != 200: exit() for item in r: From f977073808c37ea15720fd98d16e2297795c44f6 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 12 Apr 2021 16:16:31 -0400 Subject: [PATCH 085/139] Added APPEND and REPACE hydration --- CHANGE.log | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGE.log b/CHANGE.log index 7c514cc..5ad21e8 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -169,5 +169,5 @@ v2.6.9, 21 Feb 2021 -- Fixed bug in _StreamingIterable. v2.6.10, 06 Apr 2021 -- Added 'mentions' to be hydrated. -v2.7.0, 12 Apr 2021 -- Two types of hydrate: APPEND and REPLACE. +v2.7.0, 12 Apr 2021 -- Two types of hydrate: APPEND and REPLACE. Two new enums: HydrateType, OAuthType. From 2992078774990918bb0a4317d033bd754d8078e2 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Wed, 14 Apr 2021 12:40:12 -0400 Subject: [PATCH 086/139] Fixed oAuth version resolution --- CHANGE.log | 2 ++ TwitterAPI/TwitterAPI.py | 4 ++-- TwitterAPI/__init__.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index 5ad21e8..0cb8fad 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -171,3 +171,5 @@ v2.6.10, 06 Apr 2021 -- Added 'mentions' to be hydrated. v2.7.0, 12 Apr 2021 -- Two types of hydrate: APPEND and REPLACE. Two new enums: HydrateType, OAuthType. + +v2.7.1, 14 Apr 2021 -- Fixed oAuth version resolution. diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 916a675..9ddf836 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -78,7 +78,7 @@ def __init__( self.proxies = None # Twitter supports two types of authentication. - if auth_type == OAuthType.OAUTH1: + if auth_type == OAuthType.OAUTH1 or auth_type == 'oAuth1': if not all([consumer_key, consumer_secret, access_token_key, access_token_secret]): raise Exception('Missing authentication parameter') self.auth = OAuth1( @@ -86,7 +86,7 @@ def __init__( consumer_secret, access_token_key, access_token_secret) - elif auth_type == OAuthType.OAUTH2: + elif auth_type == OAuthType.OAUTH2 or auth_type == 'oAuth2': if not all([consumer_key, consumer_secret]): raise Exception('Missing authentication parameter') self.auth = OAuth2( diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index bbc8399..1d58b46 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.0' +__version__ = '2.7.1' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From ddcfba5c10bfc416e67f4b3fea62467948e3e86f Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Thu, 22 Apr 2021 13:51:54 -0400 Subject: [PATCH 087/139] Fixed _StreamIterable for v2 --- CHANGE.log | 2 ++ TwitterAPI/TwitterAPI.py | 12 ++++++------ TwitterAPI/__init__.py | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index 0cb8fad..5d0db22 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -173,3 +173,5 @@ v2.7.0, 12 Apr 2021 -- Two types of hydrate: APPEND and REPLACE. Two new enums: HydrateType, OAuthType. v2.7.1, 14 Apr 2021 -- Fixed oAuth version resolution. + +v2.7.2, 22 Apr 2021 -- Fixed _StreamIterable for v2. diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 9ddf836..e498848 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -403,12 +403,12 @@ def _iter_stream(self): break if item: item = json.loads(item.decode('utf8')) - if self.options['api_version'] == '2': - h_type = self.options['hydrate_type'] - if h_type != HydrateType.NONE: - if 'data' in item and 'includes' in item: - field_suffix = '' if h_type == HydrateType.REPLACE else '_hydrate' - item = _hydrate_tweets(item['data'], item['includes'], field_suffix) + if self.options['api_version'] == '2': + h_type = self.options['hydrate_type'] + if h_type != HydrateType.NONE: + if 'data' in item and 'includes' in item: + field_suffix = '' if h_type == HydrateType.REPLACE else '_hydrate' + item = { 'data':_hydrate_tweets(item['data'], item['includes'], field_suffix) } yield item except (ConnectionError, ValueError, ProtocolError, ReadTimeout, ReadTimeoutError, SSLError, ssl.SSLError, socket.error, json.decoder.JSONDecodeError) as e: diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 1d58b46..276801e 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.1' +__version__ = '2.7.2' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 3c91fd8222ea6c9b1cd3fc6a694b9804a378b7ce Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 26 Apr 2021 14:20:35 -0400 Subject: [PATCH 088/139] Fixed hydrate for media_keys --- CHANGE.log | 2 ++ TwitterAPI/TwitterAPI.py | 8 ++++++++ TwitterAPI/__init__.py | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGE.log b/CHANGE.log index 5d0db22..a8ba87a 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -175,3 +175,5 @@ v2.7.0, 12 Apr 2021 -- Two types of hydrate: APPEND and REPLACE. v2.7.1, 14 Apr 2021 -- Fixed oAuth version resolution. v2.7.2, 22 Apr 2021 -- Fixed _StreamIterable for v2. + +v2.7.3, 26 Apr 2021 -- Fixed hydrate for media_keys. diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index e498848..1b0ef39 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -467,5 +467,13 @@ def _create_include_fields(parent, include, new_fields): for key, value in parent.items(): if value == include[0]: new_fields.append((parent, key, include[1])) + elif isinstance(value, list) and all(isinstance(elem, str) for elem in value): + # this code gets executed for "media_keys" which are in a list object + includes_list = [] + for elem in value: + if elem == include[0]: + includes_list.append(include[1]) + if len(includes_list) > 0: + new_fields.append((parent, key, includes_list)) else: _create_include_fields(value, include, new_fields) \ No newline at end of file diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 276801e..04926c9 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.2' +__version__ = '2.7.3' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 0c66954a4cc522ef62a16aa4200c2bbc6985577b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rodrigo=20Gir=C3=A3o=20Serr=C3=A3o?= Date: Sat, 5 Jun 2021 23:10:41 +0100 Subject: [PATCH 089/139] Fix #185 Fix broken links to the docs. --- README.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 925068c..c693553 100755 --- a/README.rst +++ b/README.rst @@ -43,12 +43,12 @@ From the command line:: Documentation ------------- -* `An Introduction `_ -* `Authentication `_ -* `Error Handling `_ -* `Paging Results `_ -* `Tiny Examples `_ -* `Fault Tolerant Streams and Pages `_ +* `An Introduction `_ +* `Authentication `_ +* `Error Handling `_ +* `Paging Results `_ +* `Tiny Examples `_ +* `Fault Tolerant Streams and Pages `_ Some Code... ------------ From 2672d004dbbd0abac592806a2c598162605138af Mon Sep 17 00:00:00 2001 From: peguerosdc Date: Tue, 15 Jun 2021 02:08:52 -0500 Subject: [PATCH 090/139] Implementing v2 likes lookup --- TwitterAPI/constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index efc3930..a783253 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -241,6 +241,7 @@ 'tweets': ('GET', 'api'), 'tweets/:PARAM/hidden': ('PUT', 'api'), # ID + 'tweets/:PARAM/liking_users': ('GET', 'api'), # ID 'tweets/:PARAM': ('GET', 'api'), # ID 'tweets/compliance/jobs': ('POST', 'api'), # use method_override for 'GET' 'tweets/compliance/jobs/:PARAM': ('POST', 'api'), # ID @@ -256,6 +257,7 @@ 'users/:PARAM/following/:PARAM': ('DELETE', 'api'), # USER SOURCE ID, USER TARGET ID 'users/:PARAM/mentions': ('GET', 'api'), # ID 'users/:PARAM/tweets': ('GET', 'api'), # ID + 'users/:PARAM/liked_tweets': ('GET', 'api'), # ID 'users/by': ('GET', 'api'), 'users/by/username/:PARAM': ('GET', 'api'), # USERNAME } From c09f3dfd2012bc9959af86d56f51dc68a130a7e3 Mon Sep 17 00:00:00 2001 From: peguerosdc Date: Tue, 15 Jun 2021 02:12:43 -0500 Subject: [PATCH 091/139] Implementing v2 likes management --- TwitterAPI/constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index a783253..b7097f6 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -258,6 +258,8 @@ 'users/:PARAM/mentions': ('GET', 'api'), # ID 'users/:PARAM/tweets': ('GET', 'api'), # ID 'users/:PARAM/liked_tweets': ('GET', 'api'), # ID + 'users/:PARAM/likes': ('POST', 'api'), # ID + 'users/:PARAM/likes/:PARAM': ('DELETE', 'api'), # USER ID, TWEET ID 'users/by': ('GET', 'api'), 'users/by/username/:PARAM': ('GET', 'api'), # USERNAME } From 779659642de573e1a86cad4145d2784783af4fa7 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sun, 27 Jun 2021 08:34:45 -0400 Subject: [PATCH 092/139] Merged patches and fixed "stall_warnings" parameter --- CHANGE.log | 4 ++++ TwitterAPI/TwitterAPI.py | 4 ++-- TwitterAPI/__init__.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index a8ba87a..8865391 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -177,3 +177,7 @@ v2.7.1, 14 Apr 2021 -- Fixed oAuth version resolution. v2.7.2, 22 Apr 2021 -- Fixed _StreamIterable for v2. v2.7.3, 26 Apr 2021 -- Fixed hydrate for media_keys. + +v2.7.4, 27 Jun 2021 -- Merged RojerGS-patch-1 to fix documentation links. + Merged peguerosdc-endpoint/v2-liking to add support for V2 likes. + Changed 'stall_warning' to 'stall_warnings'. diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 1b0ef39..983a815 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -174,7 +174,7 @@ def request(self, resource, params=None, files=None, method_override=None, hydra if not params: params = {} params['delimited'] = 'length' - params['stall_warning'] = 'true' + params['stall_warnings'] = 'true' elif self.version == '2' and resource.endswith('/stream'): session.stream = True timeout = self.STREAMING_TIMEOUT @@ -476,4 +476,4 @@ def _create_include_fields(parent, include, new_fields): if len(includes_list) > 0: new_fields.append((parent, key, includes_list)) else: - _create_include_fields(value, include, new_fields) \ No newline at end of file + _create_include_fields(value, include, new_fields) diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 04926c9..8273f6f 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.3' +__version__ = '2.7.4' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 047cd3edfa53915b3b06ac4102cdcf9f2af33ac6 Mon Sep 17 00:00:00 2001 From: Justin Nebo Date: Tue, 13 Jul 2021 19:40:09 +0100 Subject: [PATCH 093/139] break if meta not in data for version 2 --- TwitterAPI/TwitterPager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py index fb3ecd6..e65b0e7 100644 --- a/TwitterAPI/TwitterPager.py +++ b/TwitterAPI/TwitterPager.py @@ -97,6 +97,8 @@ def get_iterator(self, wait=5, new_tweets=False): elif not new_tweets and item_count == 0: break else: # VERSION 2 + if 'meta' not in data: + break meta = data['meta'] if not new_tweets and not 'next_token' in meta: break From 68dd75cc12def5a5311378635306f3a57c46ecc8 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sun, 18 Jul 2021 20:25:41 -0400 Subject: [PATCH 094/139] Merged Dheavyman-fix-meta-keyerror --- CHANGE.log | 2 ++ TwitterAPI/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGE.log b/CHANGE.log index 8865391..85619db 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -181,3 +181,5 @@ v2.7.3, 26 Apr 2021 -- Fixed hydrate for media_keys. v2.7.4, 27 Jun 2021 -- Merged RojerGS-patch-1 to fix documentation links. Merged peguerosdc-endpoint/v2-liking to add support for V2 likes. Changed 'stall_warning' to 'stall_warnings'. + +v2.7.5, 18 Jul 2021 -- Merged Dheavyman-fix-meta-keyerror. diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 8273f6f..ae07a2e 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.4' +__version__ = '2.7.5' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 8fcd27082d65e073dd48b77817edaa16732f2c67 Mon Sep 17 00:00:00 2001 From: NeodymiumFerBore <32781483+NeodymiumFerBore@users.noreply.github.com> Date: Sun, 12 Sep 2021 00:49:23 +0200 Subject: [PATCH 095/139] Add comment about hydrate_type == NONE side effect Related issue: https://github.com/geduldig/TwitterAPI/issues/202#issue-993964551 --- TwitterAPI/TwitterAPI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 983a815..a6cfa25 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -150,7 +150,7 @@ def request(self, resource, params=None, files=None, method_override=None, hydra :param files: Dictionary with multipart-encoded file or None (default) :param method_override: Request method to override or None (default) :param hydrate_type: HydrateType or int - Do not hydrate - NONE or 0 (default) + Do not hydrate ('includes' field and all its content will be lost for non-streaming requests) - NONE or 0 (default) Append new field with '_hydrate' suffix with hydrate values - APPEND or 1 Replace current field value with hydrate values - REPLACE or 2 From 894c61cd2fcc6d8a2f6b5b415a25aeb702c19e42 Mon Sep 17 00:00:00 2001 From: "Rossi, Joe" Date: Sun, 26 Sep 2021 19:49:50 -0400 Subject: [PATCH 096/139] Adding V2 tweet counts API to endpoints list. --- TwitterAPI/constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index b7097f6..f7c957a 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -245,6 +245,8 @@ 'tweets/:PARAM': ('GET', 'api'), # ID 'tweets/compliance/jobs': ('POST', 'api'), # use method_override for 'GET' 'tweets/compliance/jobs/:PARAM': ('POST', 'api'), # ID + 'tweets/counts/all': ('GET', 'api'), + 'tweets/counts/recent': ('GET', 'api'), 'tweets/sample/stream': ('GET', 'api'), 'tweets/search/all': ('GET', 'api'), 'tweets/search/recent': ('GET', 'api'), From e4b33fb6feb7d0c89810f08749f92336c920561a Mon Sep 17 00:00:00 2001 From: "Rossi, Joe" Date: Tue, 28 Sep 2021 21:32:40 -0400 Subject: [PATCH 097/139] Adding blocking and muting v2 apis. --- TwitterAPI/constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index f7c957a..aff06b0 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -254,10 +254,12 @@ 'tweets/search/stream/rules': ('POST', 'api'), # use method_override for 'GET' 'users': ('GET', 'api'), 'users/:PARAM': ('GET', 'api'), # ID + 'users/:PARAM/blocking': ('GET', 'api'), # ID 'users/:PARAM/followers': ('GET', 'api'), # ID 'users/:PARAM/following': ('GET', 'api'), # ID, use method_override for 'POST' 'users/:PARAM/following/:PARAM': ('DELETE', 'api'), # USER SOURCE ID, USER TARGET ID 'users/:PARAM/mentions': ('GET', 'api'), # ID + 'users/:PARAM/muting': ('GET', 'api'), # ID 'users/:PARAM/tweets': ('GET', 'api'), # ID 'users/:PARAM/liked_tweets': ('GET', 'api'), # ID 'users/:PARAM/likes': ('POST', 'api'), # ID From c57031d2dd9ada2a95cce13abd9f68ff68c8567e Mon Sep 17 00:00:00 2001 From: Dylan Caponi Date: Wed, 21 Jul 2021 20:24:32 -0700 Subject: [PATCH 098/139] Update Ads Version "Mark your calendars for Tuesday, August 31, 2021, the end of life date for v8 of the Twitter Ads API." --- TwitterAPI/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index b7097f6..61a3c6c 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -20,7 +20,7 @@ VERSION = '1.1' CURATOR_VERSION = 'broadcast/1' -ADS_VERSION = '8' +ADS_VERSION = '9' ENDPOINTS = { From a03fdfb3637124776a88b9c410d696343de6701e Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Thu, 11 Nov 2021 13:41:09 -0800 Subject: [PATCH 099/139] Merged dylancaponi-patch-2 to update ADS version --- CHANGE.log | 2 ++ TwitterAPI/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGE.log b/CHANGE.log index 85619db..cfaa2b5 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -183,3 +183,5 @@ v2.7.4, 27 Jun 2021 -- Merged RojerGS-patch-1 to fix documentation links. Changed 'stall_warning' to 'stall_warnings'. v2.7.5, 18 Jul 2021 -- Merged Dheavyman-fix-meta-keyerror. + +v2.7.6, 11 Nov 2021 -- Merged dylancaponi-patch-2 to update ADS version. diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index ae07a2e..16c7e4c 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.5' +__version__ = '2.7.6' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 904b9f2ebafe24e28eed2f91dc08715f0d88ab6e Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Thu, 11 Nov 2021 14:43:21 -0800 Subject: [PATCH 100/139] Merged NeodymiumFerBore-patch-1, musicist288-master --- CHANGE.log | 2 ++ TwitterAPI/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGE.log b/CHANGE.log index cfaa2b5..8a92afb 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -185,3 +185,5 @@ v2.7.4, 27 Jun 2021 -- Merged RojerGS-patch-1 to fix documentation links. v2.7.5, 18 Jul 2021 -- Merged Dheavyman-fix-meta-keyerror. v2.7.6, 11 Nov 2021 -- Merged dylancaponi-patch-2 to update ADS version. + +v2.7.7, 11 Nov 2021 -- Merged NeodymiumFerBore-patch-1, musicist288-master \ No newline at end of file diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 16c7e4c..b79225d 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.6' +__version__ = '2.7.7' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 9fb856c383737cc379f97ef4026cf9b89b076e15 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 12 Nov 2021 10:39:54 -0800 Subject: [PATCH 101/139] 2.7 docs --- docs/README.rst | 19 +++++++++++++++---- docs/conf.py | 4 ++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/README.rst b/docs/README.rst index ab14108..39ff422 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -1,25 +1,36 @@ Instructions for Generating Documentation with Sphinx ===================================================== +INSTALL SPHINX:: + + pip3 install -U Sphinx + MAKE NEW DOCS IN MASTER BRANCH:: git clone git@github.com:geduldig/TwitterAPI.git cd TwitterAPI/docs + vi conf.py (modify version and release) make html +TEST DOC WEB PAGE:: + + TwitterAPI/docs/_build/html/index.html + DOWNLOAD OLD DOCS FROM GH-PAGES BRANCH:: - cd gh-pages + cd .. + mkdir TwitterAPI-docs + cd TwitterAPI-docs git clone -b gh-pages git@github.com:geduldig/TwitterAPI.git COPY NEW DOCS OVER OLD DOCS:: - cp TwitterAPI/docs/_build/html/*.* gh-pages/TwitterAPI - cp -r TwitterAPI/docs/_build/html/* gh-pages/TwitterAPI + cd .. + cp TwitterAPI/docs/_build/html/*.* TwitterAPI-docs/TwitterAPI + cp -r TwitterAPI/docs/_build/html/* TwitterAPI-docs/TwitterAPI rm -r TwitterAPI/docs/_build UPLOAD NEW DOCS TO GH-PAGES BRANCH::: - cd gh-pages/TwitterAPI git commit -am "2.5 docs" git push origin gh-pages diff --git a/docs/conf.py b/docs/conf.py index 0da6b7a..2819b98 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,9 +56,9 @@ # built documents. # # The short X.Y version. -version = '2.6' +version = '2.7' # The full version, including alpha/beta/rc tags. -release = '2.6' +release = '2.7' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 358ef6a32bcc09b41d6716e02a2a30fac8bf4f5c Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 12 Nov 2021 10:47:40 -0800 Subject: [PATCH 102/139] Updated docs README --- TwitterAPI/__init__.py | 2 +- docs/README.rst | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index b79225d..5c37e02 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.7' +__version__ = '2.7.7.1' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/docs/README.rst b/docs/README.rst index 39ff422..07bd43e 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -18,19 +18,20 @@ TEST DOC WEB PAGE:: DOWNLOAD OLD DOCS FROM GH-PAGES BRANCH:: - cd .. - mkdir TwitterAPI-docs - cd TwitterAPI-docs git clone -b gh-pages git@github.com:geduldig/TwitterAPI.git COPY NEW DOCS OVER OLD DOCS:: - cd .. - cp TwitterAPI/docs/_build/html/*.* TwitterAPI-docs/TwitterAPI - cp -r TwitterAPI/docs/_build/html/* TwitterAPI-docs/TwitterAPI - rm -r TwitterAPI/docs/_build + cp ./_build/html/*.* ./TwitterAPI + cp -r ./_build/html/* ./TwitterAPI + rm -r ./_build UPLOAD NEW DOCS TO GH-PAGES BRANCH::: + cd TwitterAPI git commit -am "2.5 docs" git push origin gh-pages + +CLEAN UP:: + cd .. + rm -r TwitterAPI \ No newline at end of file From ad1b020206aef4ce7f3af43dccbc15b08565052c Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 12 Nov 2021 11:08:15 -0800 Subject: [PATCH 103/139] Updated badges --- README.rst | 4 ++-- TwitterAPI/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index c693553..24fd79d 100755 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ ============================ |BADGE_VERSION| |BADGE_CHAT| ============================ -|BADGE_1.1| |BADGE_PREMIUM| |BADGE_2| |BADGE_ADS8| |BADGE_LABS| +|BADGE_1.1| |BADGE_PREMIUM| |BADGE_2| |BADGE_ADS| |BADGE_LABS| ============================ .. |LOGO| image:: https://raw.githubusercontent.com/geduldig/TwitterAPI/master/logo.png @@ -16,7 +16,7 @@ :target: https://developer.twitter.com/en/docs/twitter-api .. |BADGE_LABS| image:: https://img.shields.io/endpoint?url=https%3A%2F%2Fblue-sea-697d.quartiers047.workers.dev%3A443%2Fhttps%2Ftwbadges.glitch.me%2Fbadges%2Flabs :target: https://developer.twitter.com/en/docs/labs -.. |BADGE_ADS8| image:: https://img.shields.io/endpoint?url=https%3A%2F%2Fblue-sea-697d.quartiers047.workers.dev%3A443%2Fhttps%2Ftwbadges.glitch.me%2Fbadges%2Fadsv8 +.. |BADGE_ADS| image:: https://img.shields.io/endpoint?url=https%3A%2F%2Fblue-sea-697d.quartiers047.workers.dev%3A443%2Fhttps%2Ftwbadges.glitch.me%2Fbadges%2Fadsv9 :target: https://developer.twitter.com/en/docs/twitter-ads-api .. |BADGE_1.1| image:: https://img.shields.io/endpoint?url=https%3A%2F%2Fblue-sea-697d.quartiers047.workers.dev%3A443%2Fhttps%2Ftwbadges.glitch.me%2Fbadges%2Fstandard :target: https://developer.twitter.com/en/docs/twitter-api diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 5c37e02..85a6394 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.7.1' +__version__ = '2.7.7.2' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From f0687ca55e042fc2403f0c88abc1da8afe4d765b Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 29 Nov 2021 20:40:24 -0500 Subject: [PATCH 104/139] Fixed hydrated media keys --- CHANGE.log | 4 +++- README.rst | 22 +++++++++++----------- TwitterAPI/TwitterAPI.py | 11 ++++++++++- TwitterAPI/__init__.py | 2 +- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index 8a92afb..05ea323 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -186,4 +186,6 @@ v2.7.5, 18 Jul 2021 -- Merged Dheavyman-fix-meta-keyerror. v2.7.6, 11 Nov 2021 -- Merged dylancaponi-patch-2 to update ADS version. -v2.7.7, 11 Nov 2021 -- Merged NeodymiumFerBore-patch-1, musicist288-master \ No newline at end of file +v2.7.7, 11 Nov 2021 -- Merged NeodymiumFerBore-patch-1, musicist288-master + +v2.7.8, 29 Nov 2021 -- Fixed hydrated media keys \ No newline at end of file diff --git a/README.rst b/README.rst index 24fd79d..24002e8 100755 --- a/README.rst +++ b/README.rst @@ -41,15 +41,6 @@ From the command line:: pip install TwitterAPI -Documentation -------------- -* `An Introduction `_ -* `Authentication `_ -* `Error Handling `_ -* `Paging Results `_ -* `Tiny Examples `_ -* `Fault Tolerant Streams and Pages `_ - Some Code... ------------ [See `TwitterAPI/examples `_ for working examples.] @@ -81,9 +72,18 @@ Stream tweets from New York City:: for item in r: print(item) -Notice that ``request()`` works with all endpoints found in either the REST APIs or the Streaming APIs. Usually ``request()`` takes two arguments: a Twitter endpoint and a dictionary of endpoint parameters. The above examples use ``get_iterator()`` to consume each tweet object. The iterator knows how to iterate results returned from either the REST APIs or the Streaming APIs. +``request()`` works with all version 1.1 and version 2 endpoints and the other supported Twitter APIs. Typcally, ``request()`` takes two arguments: a Twitter endpoint and a dictionary of endpoint parameters. The above examples use ``get_iterator()`` to consume each tweet object. The iterator can iterate results returned from either the REST APIs or the Streaming APIs. -You also have access to the response object returned by ``request()``. From a response object ``r`` you can get the raw response with ``r.text`` and the HTTP status code with ``r.status_code``. See the `requests `_ library documentation for more details. +You also have access to the response object returned by ``request()``. With a response object ``r`` comes the raw response (``r.text``) and the HTTP status code (``r.status_code``). See the `requests `_ library documentation for more details. + +Documentation +------------- +* `An Introduction `_ +* `Authentication `_ +* `Error Handling `_ +* `Paging Results `_ +* `Tiny Examples `_ +* `Fault Tolerant Streams and Pages `_ Extra Goodies ------------- diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index a6cfa25..86d23c4 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -452,7 +452,16 @@ def _hydrate_tweets(data, includes, field_suffix): parent = item[0] field = item[1] + field_suffix include = item[2] - parent[field] = include + if field in parent: + if item[1] == 'media_keys': + parent[field] += include + if field_suffix == '': + # REPLACE option + parent[field].remove(include[0]['media_key']) + else: + parent[field] = include + else: + parent[field] = include return data diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 85a6394..53c86fc 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.7.2' +__version__ = '2.7.8' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From c27556bf0401a7c3caee04e15d997b655f90868e Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Tue, 30 Nov 2021 17:50:57 -0500 Subject: [PATCH 105/139] Specify all supported endpoint methods in constants.py --- CHANGE.log | 4 +++- TwitterAPI/__init__.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index 05ea323..131cd22 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -188,4 +188,6 @@ v2.7.6, 11 Nov 2021 -- Merged dylancaponi-patch-2 to update ADS version. v2.7.7, 11 Nov 2021 -- Merged NeodymiumFerBore-patch-1, musicist288-master -v2.7.8, 29 Nov 2021 -- Fixed hydrated media keys \ No newline at end of file +v2.7.8, 29 Nov 2021 -- Fixed hydrated media keys + +v2.7.9, 30 Nov 2021 -- Specify all supported endpoint methods in constants.py diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 53c86fc..895948c 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.8' +__version__ = '2.7.9' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 5be49297570fecb455ba12d1df36fe4f1c46fa2d Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Tue, 30 Nov 2021 18:26:23 -0500 Subject: [PATCH 106/139] Added new v2 endpoints --- CHANGE.log | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGE.log b/CHANGE.log index 131cd22..7fdeab3 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -191,3 +191,4 @@ v2.7.7, 11 Nov 2021 -- Merged NeodymiumFerBore-patch-1, musicist288-master v2.7.8, 29 Nov 2021 -- Fixed hydrated media keys v2.7.9, 30 Nov 2021 -- Specify all supported endpoint methods in constants.py + Added new v2 endpoints From ebf5643cff31653fcc318628797615501bb87a7c Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Tue, 30 Nov 2021 18:36:40 -0500 Subject: [PATCH 107/139] Added new v2 endpoints --- TwitterAPI/TwitterAPI.py | 29 ++- TwitterAPI/__init__.py | 2 +- TwitterAPI/constants.py | 460 +++++++++++++++++++++------------------ 3 files changed, 266 insertions(+), 225 deletions(-) diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 86d23c4..8d88a42 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -148,7 +148,8 @@ def request(self, resource, params=None, files=None, method_override=None, hydra :param resource: A Twitter endpoint (ex. "search/tweets") :param params: Dictionary with endpoint parameters or None (default) :param files: Dictionary with multipart-encoded file or None (default) - :param method_override: Request method to override or None (default) + :param method_override: Request method to override or None (default). + If an endpoint supports more than one method, the default method is the first in the list of methods. :param hydrate_type: HydrateType or int Do not hydrate ('includes' field and all its content will be lost for non-streaming requests) - NONE or 0 (default) Append new field with '_hydrate' suffix with hydrate values - APPEND or 1 @@ -157,24 +158,34 @@ def request(self, resource, params=None, files=None, method_override=None, hydra :returns: TwitterResponse :raises: TwitterConnectionError """ + # check that the endpoint is valid resource, endpoint = self._get_endpoint(resource) if endpoint not in ENDPOINTS: raise Exception('Endpoint "%s" unsupported' % endpoint) + # check that the method is valid if the endpoint supports more than one method + method, subdomain = ENDPOINTS[endpoint] + if not method_override and isinstance(method, list): + method = method[0] # by default use first method in list + elif isinstance(method, list): + if method_override in method: + method = method_override # use method_override + else: + raise Exception(f'Endpoint "{endpoint}" with method "{method_override}" unsupported') + + if not params: + params = {} + if self.version == '1.1': + # used only for streaming + params['delimited'] = 'length' + params['stall_warnings'] = 'true' + with requests.Session() as session: session.auth = self.auth session.headers = {'User-Agent': self.USER_AGENT} - method, subdomain = ENDPOINTS[endpoint] - if method_override: - method = method_override url = self._prepare_url(subdomain, resource) if self.version == '1.1' and 'stream' in subdomain: session.stream = True timeout = self.STREAMING_TIMEOUT - # always use 'delimited' for efficient stream parsing - if not params: - params = {} - params['delimited'] = 'length' - params['stall_warnings'] = 'true' elif self.version == '2' and resource.endswith('/stream'): session.stream = True timeout = self.STREAMING_TIMEOUT diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 895948c..646f621 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.9' +__version__ = '2.7.9.1' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 717e7a4..adb65f2 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -24,246 +24,276 @@ ENDPOINTS = { - # resource: (method, subdomain) + # resource: (method(s), subdomain) # STREAMING API - 'statuses/filter': ('POST', 'stream'), - 'statuses/firehose': ('GET', 'stream'), - 'statuses/sample': ('GET', 'stream'), - 'site': ('GET', 'sitestream'), - 'user': ('GET', 'userstream'), + 'statuses/filter': ('POST', 'stream'), + 'statuses/firehose': ('GET', 'stream'), + 'statuses/sample': ('GET', 'stream'), + 'site': ('GET', 'sitestream'), + 'user': ('GET', 'userstream'), # PUBLIC API - 'account/remove_profile_banner': ('POST', 'api'), - 'account/settings': ('GET', 'api'), - 'account/update_delivery_device': ('POST', 'api'), - 'account/update_profile': ('POST', 'api'), - 'account/update_profile_background_image': ('POST', 'api'), - 'account/update_profile_banner': ('POST', 'api'), - 'account/update_profile_colors': ('POST', 'api'), - 'account/update_profile_image': ('POST', 'api'), - 'account/verify_credentials': ('GET', 'api'), + 'account/remove_profile_banner': ('POST', 'api'), + 'account/settings': ('GET', 'api'), + 'account/update_delivery_device': ('POST', 'api'), + 'account/update_profile': ('POST', 'api'), + 'account/update_profile_background_image': ('POST', 'api'), + 'account/update_profile_banner': ('POST', 'api'), + 'account/update_profile_colors': ('POST', 'api'), + 'account/update_profile_image': ('POST', 'api'), + 'account/verify_credentials': ('GET', 'api'), - 'application/rate_limit_status': ('GET', 'api'), - - 'blocks/create': ('POST', 'api'), - 'blocks/destroy': ('POST', 'api'), - 'blocks/ids': ('GET', 'api'), - 'blocks/list': ('GET', 'api'), - - 'direct_messages': ('GET', 'api'), # deprecated - 'direct_messages/destroy': ('POST', 'api'), # deprecated - 'direct_messages/events/destroy': ('DELETE', 'api'), - 'direct_messages/events/new': ('POST', 'api'), - 'direct_messages/events/list': ('GET', 'api'), - 'direct_messages/events/show': ('GET', 'api'), - 'direct_messages/indicate_typing': ('POST', 'api'), - 'direct_messages/new': ('POST', 'api'), # deprecated - 'direct_messages/sent': ('GET', 'api'), # deprecated - 'direct_messages/show': ('GET', 'api'), # deprecated - 'direct_messages/welcome_messages/new': ('POST', 'api'), - 'direct_messages/welcome_messages/list': ('GET', 'api'), - 'direct_messages/welcome_messages/show': ('GET', 'api'), - 'direct_messages/welcome_messages/destroy': ('DELETE', 'api'), - 'direct_messages/welcome_messages/rules/new': ('POST', 'api'), - 'direct_messages/welcome_messages/rules/list': ('GET', 'api'), - 'direct_messages/welcome_messages/rules/show': ('GET', 'api'), - 'direct_messages/welcome_messages/rules/destroy': ('DELETE', 'api'), - - 'favorites/create': ('POST', 'api'), - 'favorites/destroy': ('POST', 'api'), - 'favorites/list': ('GET', 'api'), - - 'followers/ids': ('GET', 'api'), - 'followers/list': ('GET', 'api'), - - 'friends/ids': ('GET', 'api'), - 'friends/list': ('GET', 'api'), - - 'friendships/create': ('POST', 'api'), - 'friendships/destroy': ('POST', 'api'), - 'friendships/incoming': ('GET', 'api'), - 'friendships/lookup': ('GET', 'api'), - 'friendships/no_retweets/ids': ('GET', 'api'), - 'friendships/outgoing': ('GET', 'api'), - 'friendships/show': ('GET', 'api'), - 'friendships/update': ('POST', 'api'), - - 'lists/create': ('POST', 'api'), - 'lists/destroy': ('POST', 'api'), - 'lists/list': ('GET', 'api'), - 'lists/members': ('GET', 'api'), - 'lists/members/create': ('POST', 'api'), - 'lists/members/create_all': ('POST', 'api'), - 'lists/members/destroy': ('POST', 'api'), - 'lists/members/destroy_all': ('POST', 'api'), - 'lists/members/show': ('GET', 'api'), - 'lists/memberships': ('GET', 'api'), - 'lists/ownerships': ('GET', 'api'), - 'lists/show': ('GET', 'api'), - 'lists/statuses': ('GET', 'api'), - 'lists/subscribers': ('GET', 'api'), - 'lists/subscribers/create': ('POST', 'api'), - 'lists/subscribers/destroy': ('POST', 'api'), - 'lists/subscribers/show': ('GET', 'api'), - 'lists/subscriptions': ('GET', 'api'), - 'lists/update': ('POST', 'api'), - - 'media/metadata/create': ('POST', 'upload'), - 'media/upload': ('POST', 'upload'), # use method_override for 'GET' - 'media/subtitles/create': ('POST', 'upload'), - 'media/subtitles/delete': ('POST', 'upload'), - - 'mutes/users/create': ('POST', 'api'), - 'mutes/users/destroy': ('POST', 'api'), - 'mutes/users/ids': ('GET', 'api'), - 'mutes/users/list': ('GET', 'api'), - - 'geo/id/:PARAM': ('GET', 'api'), # PLACE_ID - 'geo/place': ('POST', 'api'), - 'geo/reverse_geocode': ('GET', 'api'), - 'geo/search': ('GET', 'api'), - 'geo/similar_places': ('GET', 'api'), - - 'help/configuration': ('GET', 'api'), - 'help/languages': ('GET', 'api'), - 'help/privacy': ('GET', 'api'), - 'help/tos': ('GET', 'api'), - - 'saved_searches/create': ('POST', 'api'), - 'saved_searches/destroy/:PARAM': ('POST', 'api'), # ID - 'saved_searches/list': ('GET', 'api'), - 'saved_searches/show/:PARAM': ('GET', 'api'), # ID - - 'search/tweets': ('GET', 'api'), - - 'statuses/destroy/:PARAM': ('POST', 'api'), # ID - 'statuses/home_timeline': ('GET', 'api'), - 'statuses/lookup': ('GET', 'api'), - 'statuses/mentions_timeline': ('GET', 'api'), - 'statuses/oembed': ('GET', 'api'), - 'statuses/retweet/:PARAM': ('POST', 'api'), # ID - 'statuses/retweeters/ids': ('GET', 'api'), - 'statuses/retweets/:PARAM': ('GET', 'api'), # ID - 'statuses/retweets_of_me': ('GET', 'api'), - 'statuses/show/:PARAM': ('GET', 'api'), # ID - 'statuses/unretweet/:PARAM': ('POST', 'api'), # ID - 'statuses/user_timeline': ('GET', 'api'), - 'statuses/update': ('POST', 'api'), - 'statuses/update_with_media': ('POST', 'api'), # deprecated - - 'trends/available': ('GET', 'api'), - 'trends/closest': ('GET', 'api'), - 'trends/place': ('GET', 'api'), - - 'users/contributees': ('GET', 'api'), - 'users/contributors': ('GET', 'api'), - 'users/lookup': ('POST', 'api'), - 'users/profile_banner': ('GET' 'api'), - 'users/report_spam': ('POST', 'api'), - 'users/search': ('GET', 'api'), - 'users/show': ('GET', 'api'), - 'users/suggestions': ('GET', 'api'), - 'users/suggestions/:PARAM': ('GET', 'api'), # SLUG - 'users/suggestions/:PARAM/members': ('GET', 'api'), # SLUG + 'application/rate_limit_status': ('GET', 'api'), + + 'blocks/create': ('POST', 'api'), + 'blocks/destroy': ('POST', 'api'), + 'blocks/ids': ('GET', 'api'), + 'blocks/list': ('GET', 'api'), + + 'direct_messages': ('GET', 'api'), # deprecated + 'direct_messages/destroy': ('POST', 'api'), # deprecated + 'direct_messages/events/destroy': ('DELETE', 'api'), + 'direct_messages/events/new': ('POST', 'api'), + 'direct_messages/events/list': ('GET', 'api'), + 'direct_messages/events/show': ('GET', 'api'), + 'direct_messages/indicate_typing': ('POST', 'api'), + 'direct_messages/new': ('POST', 'api'), # deprecated + 'direct_messages/sent': ('GET', 'api'), # deprecated + 'direct_messages/show': ('GET', 'api'), # deprecated + 'direct_messages/welcome_messages/new': ('POST', 'api'), + 'direct_messages/welcome_messages/list': ('GET', 'api'), + 'direct_messages/welcome_messages/show': ('GET', 'api'), + 'direct_messages/welcome_messages/destroy': ('DELETE', 'api'), + 'direct_messages/welcome_messages/rules/new': ('POST', 'api'), + 'direct_messages/welcome_messages/rules/list': ('GET', 'api'), + 'direct_messages/welcome_messages/rules/show': ('GET', 'api'), + 'direct_messages/welcome_messages/rules/destroy': ('DELETE', 'api'), + + 'favorites/create': ('POST', 'api'), + 'favorites/destroy': ('POST', 'api'), + 'favorites/list': ('GET', 'api'), + + 'followers/ids': ('GET', 'api'), + 'followers/list': ('GET', 'api'), + + 'friends/ids': ('GET', 'api'), + 'friends/list': ('GET', 'api'), + + 'friendships/create': ('POST', 'api'), + 'friendships/destroy': ('POST', 'api'), + 'friendships/incoming': ('GET', 'api'), + 'friendships/lookup': ('GET', 'api'), + 'friendships/no_retweets/ids': ('GET', 'api'), + 'friendships/outgoing': ('GET', 'api'), + 'friendships/show': ('GET', 'api'), + 'friendships/update': ('POST', 'api'), + + 'lists/create': ('POST', 'api'), + 'lists/destroy': ('POST', 'api'), + 'lists/list': ('GET', 'api'), + 'lists/members': ('GET', 'api'), + 'lists/members/create': ('POST', 'api'), + 'lists/members/create_all': ('POST', 'api'), + 'lists/members/destroy': ('POST', 'api'), + 'lists/members/destroy_all': ('POST', 'api'), + 'lists/members/show': ('GET', 'api'), + 'lists/memberships': ('GET', 'api'), + 'lists/ownerships': ('GET', 'api'), + 'lists/show': ('GET', 'api'), + 'lists/statuses': ('GET', 'api'), + 'lists/subscribers': ('GET', 'api'), + 'lists/subscribers/create': ('POST', 'api'), + 'lists/subscribers/destroy': ('POST', 'api'), + 'lists/subscribers/show': ('GET', 'api'), + 'lists/subscriptions': ('GET', 'api'), + 'lists/update': ('POST', 'api'), + + 'media/metadata/create': ('POST', 'upload'), + 'media/upload': (['POST','GET'], 'upload'), + 'media/subtitles/create': ('POST', 'upload'), + 'media/subtitles/delete': ('POST', 'upload'), + + 'mutes/users/create': ('POST', 'api'), + 'mutes/users/destroy': ('POST', 'api'), + 'mutes/users/ids': ('GET', 'api'), + 'mutes/users/list': ('GET', 'api'), + + 'geo/id/:PARAM': ('GET', 'api'), # PLACE_ID + 'geo/place': ('POST', 'api'), + 'geo/reverse_geocode': ('GET', 'api'), + 'geo/search': ('GET', 'api'), + 'geo/similar_places': ('GET', 'api'), + + 'help/configuration': ('GET', 'api'), + 'help/languages': ('GET', 'api'), + 'help/privacy': ('GET', 'api'), + 'help/tos': ('GET', 'api'), + + 'saved_searches/create': ('POST', 'api'), + 'saved_searches/destroy/:PARAM': ('POST', 'api'), # ID + 'saved_searches/list': ('GET', 'api'), + 'saved_searches/show/:PARAM': ('GET', 'api'), # ID + + 'search/tweets': ('GET', 'api'), + + 'statuses/destroy/:PARAM': ('POST', 'api'), # ID + 'statuses/home_timeline': ('GET', 'api'), + 'statuses/lookup': ('GET', 'api'), + 'statuses/mentions_timeline': ('GET', 'api'), + 'statuses/oembed': ('GET', 'api'), + 'statuses/retweet/:PARAM': ('POST', 'api'), # ID + 'statuses/retweeters/ids': ('GET', 'api'), + 'statuses/retweets/:PARAM': ('GET', 'api'), # ID + 'statuses/retweets_of_me': ('GET', 'api'), + 'statuses/show/:PARAM': ('GET', 'api'), # ID + 'statuses/unretweet/:PARAM': ('POST', 'api'), # ID + 'statuses/user_timeline': ('GET', 'api'), + 'statuses/update': ('POST', 'api'), + 'statuses/update_with_media': ('POST', 'api'), # deprecated + + 'trends/available': ('GET', 'api'), + 'trends/closest': ('GET', 'api'), + 'trends/place': ('GET', 'api'), + + 'users/contributees': ('GET', 'api'), + 'users/contributors': ('GET', 'api'), + 'users/lookup': ('POST', 'api'), + 'users/profile_banner': ('GET' 'api'), + 'users/report_spam': ('POST', 'api'), + 'users/search': ('GET', 'api'), + 'users/show': ('GET', 'api'), + 'users/suggestions': ('GET', 'api'), + 'users/suggestions/:PARAM': ('GET', 'api'), # SLUG + 'users/suggestions/:PARAM/members': ('GET', 'api'), # SLUG # COLLECTIONS API - 'collections/create': ('POST', 'api'), - 'collections/destroy': ('POST', 'api'), - 'collections/entries': ('GET', 'api'), - 'collections/entries/add': ('POST', 'api'), - 'collections/entries/curate': ('POST', 'api'), - 'collections/entries/move': ('POST', 'api'), - 'collections/entries/remove': ('POST', 'api'), - 'collections/list': ('GET', 'api'), - 'collections/show': ('GET', 'api'), - 'collections/update': ('POST', 'api'), + 'collections/create': ('POST', 'api'), + 'collections/destroy': ('POST', 'api'), + 'collections/entries': ('GET', 'api'), + 'collections/entries/add': ('POST', 'api'), + 'collections/entries/curate': ('POST', 'api'), + 'collections/entries/move': ('POST', 'api'), + 'collections/entries/remove': ('POST', 'api'), + 'collections/list': ('GET', 'api'), + 'collections/show': ('GET', 'api'), + 'collections/update': ('POST', 'api'), # CURATOR API - 'collections/:PARAM/content': ('GET', 'curator'), # ID - 'projects': ('GET', 'curator'), - 'projects/:PARAM': ('GET', 'curator'), # ID - 'streams/:PARAM/content': ('GET', 'curator'), # ID - 'streams/:PARAM/metrics': ('GET', 'curator'), # ID - 'streams/:PARAM/trendline': ('GET', 'curator'), # ID - 'streams/compare': ('GET', 'curator'), - 'streams/compare_to_target': ('GET', 'curator'), + 'collections/:PARAM/content': ('GET', 'curator'), # ID + 'projects': ('GET', 'curator'), + 'projects/:PARAM': ('GET', 'curator'), # ID + 'streams/:PARAM/content': ('GET', 'curator'), # ID + 'streams/:PARAM/metrics': ('GET', 'curator'), # ID + 'streams/:PARAM/trendline': ('GET', 'curator'), # ID + 'streams/compare': ('GET', 'curator'), + 'streams/compare_to_target': ('GET', 'curator'), # ADS API (not tested!!) - 'accounts/:PARAM/auction_insights': ('GET', 'ads-api'), # ACCOUNT ID - 'accounts/:PARAM/promoted_tweets': ('GET', 'ads-api'), # ACCOUNT ID - 'stats/accounts/:PARAM': ('GET', 'ads-api'), # ACCOUNT ID - 'stats/accounts/:PARAM/reach/funding_instruments': ('GET', 'ads-api'), # ACCOUNT ID - 'stats/jobs/accounts/:PARAM': ('GET', 'ads-api'), # ACCOUNT ID - 'stats/jobs/accounts/:PARAM': ('POST', 'ads-api'), # ACCOUNT ID - 'stats/jobs/accounts/:PARAM/:PARAM': ('DELETE', 'ads-api'), # ACCOUNT ID, JOB ID - 'stats/jobs/summaries': ('GET', 'ads-api'), + 'accounts/:PARAM/auction_insights': ('GET', 'ads-api'), # ACCOUNT ID + 'accounts/:PARAM/promoted_tweets': ('GET', 'ads-api'), # ACCOUNT ID + 'stats/accounts/:PARAM': ('GET', 'ads-api'), # ACCOUNT ID + 'stats/accounts/:PARAM/reach/funding_instruments': ('GET', 'ads-api'), # ACCOUNT ID + 'stats/jobs/accounts/:PARAM': ('GET', 'ads-api'), # ACCOUNT ID + 'stats/jobs/accounts/:PARAM': ('POST', 'ads-api'), # ACCOUNT ID + 'stats/jobs/accounts/:PARAM/:PARAM': ('DELETE', 'ads-api'), # ACCOUNT ID, JOB ID + 'stats/jobs/summaries': ('GET', 'ads-api'), # ACCOUNT ACTIVITY WEBHOOK API - 'account_activity/all/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/subscriptions': ('DELETE', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/subscriptions/all': ('GET', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/subscriptions/all/list': ('GET', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/webhooks': ('POST', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/webhooks/:PARAM': ('DELETE', 'api'), # ENVIRONMENT NAME, WEBHOOK ID - 'account_activity/all/count': ('GET', 'api'), - 'account_activity/all/webhooks': ('GET', 'api'), - 'account_activity/webhooks': ('POST', 'api'), - 'account_activity/webhooks/:PARAM': ('DELETE', 'api'), # WEBHOOK ID - 'account_activity/webhooks/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME - 'account_activity/webhooks/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions': ('DELETE', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions/all': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions/all/list': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/webhooks': ('POST', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/webhooks/:PARAM': ('DELETE', 'api'), # ENVIRONMENT NAME, WEBHOOK ID + 'account_activity/all/count': ('GET', 'api'), + 'account_activity/all/webhooks': ('GET', 'api'), + 'account_activity/webhooks': ('POST', 'api'), + 'account_activity/webhooks/:PARAM': ('DELETE', 'api'), # WEBHOOK ID + 'account_activity/webhooks/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME + 'account_activity/webhooks/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME # PREMIUM SEARCH API - 'tweets/search/30day/:PARAM': ('GET', 'api'), # LABEL - 'tweets/search/30day/:PARAM/counts': ('GET', 'api'), # LABEL - 'tweets/search/fullarchive/:PARAM': ('GET', 'api'), # LABEL - 'tweets/search/fullarchive/:PARAM/counts': ('GET', 'api'), # LABEL + 'tweets/search/30day/:PARAM': ('GET', 'api'), # LABEL + 'tweets/search/30day/:PARAM/counts': ('GET', 'api'), # LABEL + 'tweets/search/fullarchive/:PARAM': ('GET', 'api'), # LABEL + 'tweets/search/fullarchive/:PARAM/counts': ('GET', 'api'), # LABEL # LABS API (BETAS) WILL NEED APPLICATION APPROVAL - 'labs/1/tweets/metrics/private': ('GET', 'api'), - 'labs/2/tweets/:PARAM': ('GET', 'api'), # TWEET ID - 'labs/2/tweets': ('GET', 'api'), - 'labs/2/tweets/search': ('GET', 'api'), - 'labs/2/tweets/:PARAM/hidden': ('PUT', 'api'), # TWEET ID - 'labs/2/users/:PARAM': ('GET', 'api'), # USER ID - 'labs/2/users': ('GET', 'api'), + 'labs/1/tweets/metrics/private': ('GET', 'api'), + 'labs/2/tweets/:PARAM': ('GET', 'api'), # TWEET ID + 'labs/2/tweets': ('GET', 'api'), + 'labs/2/tweets/search': ('GET', 'api'), + 'labs/2/tweets/:PARAM/hidden': ('PUT', 'api'), # TWEET ID + 'labs/2/users/:PARAM': ('GET', 'api'), # USER ID + 'labs/2/users': ('GET', 'api'), + # !!!!!!!!!!!!!!!!!!! # API V2 EARLY ACCESS - - 'tweets': ('GET', 'api'), - 'tweets/:PARAM/hidden': ('PUT', 'api'), # ID - 'tweets/:PARAM/liking_users': ('GET', 'api'), # ID - 'tweets/:PARAM': ('GET', 'api'), # ID - 'tweets/compliance/jobs': ('POST', 'api'), # use method_override for 'GET' - 'tweets/compliance/jobs/:PARAM': ('POST', 'api'), # ID - 'tweets/counts/all': ('GET', 'api'), - 'tweets/counts/recent': ('GET', 'api'), - 'tweets/sample/stream': ('GET', 'api'), - 'tweets/search/all': ('GET', 'api'), - 'tweets/search/recent': ('GET', 'api'), - 'tweets/search/stream': ('GET', 'api'), - 'tweets/search/stream/rules': ('POST', 'api'), # use method_override for 'GET' - 'users': ('GET', 'api'), - 'users/:PARAM': ('GET', 'api'), # ID - 'users/:PARAM/blocking': ('GET', 'api'), # ID - 'users/:PARAM/followers': ('GET', 'api'), # ID - 'users/:PARAM/following': ('GET', 'api'), # ID, use method_override for 'POST' - 'users/:PARAM/following/:PARAM': ('DELETE', 'api'), # USER SOURCE ID, USER TARGET ID - 'users/:PARAM/mentions': ('GET', 'api'), # ID - 'users/:PARAM/muting': ('GET', 'api'), # ID - 'users/:PARAM/tweets': ('GET', 'api'), # ID - 'users/:PARAM/liked_tweets': ('GET', 'api'), # ID - 'users/:PARAM/likes': ('POST', 'api'), # ID - 'users/:PARAM/likes/:PARAM': ('DELETE', 'api'), # USER ID, TWEET ID - 'users/by': ('GET', 'api'), - 'users/by/username/:PARAM': ('GET', 'api'), # USERNAME + # !!!!!!!!!!!!!!!!!!! + + 'compliance/jobs': (['GET','POST'], 'api'), + 'compliance/jobs/:PARAM': ('GET', 'api'), # ID + + 'lists': ('POST', 'api'), + 'lists/:PARAM': (['GET','PUT','DELETE'], 'api'), # ID + 'lists/:PARAM/followers': ('GET', 'api'), + 'lists/:PARAM/members': (['GET','POST'], 'api'), # ID + 'lists/:PARAM/members/:PARAM': ('DELETE', 'api'), # ID, USER ID + 'lists/:PARAM/tweets': ('GET', 'api'), # ID + + 'spaces': ('GET', 'api'), + 'spaces/:PARAM': ('GET', 'api'), # ID + 'spaces/:PARAM/buyers': ('GET', 'api'), # ID + 'spaces/by/creator_ids': ('GET', 'api'), + 'spaces/search': ('GET', 'api'), + + 'tweets': (['GET','POST'], 'api'), + 'tweets/:PARAM': (['GET','DELETE'], 'api'), # ID + 'tweets/:PARAM/hidden': ('PUT', 'api'), # ID + 'tweets/:PARAM/liking_users': ('GET', 'api'), # ID + 'tweets/:PARAM/retweeted_by': ('GET', 'api'), # ID + 'tweets/counts/all': ('GET', 'api'), + 'tweets/counts/recent': ('GET', 'api'), + 'tweets/sample/stream': ('GET', 'api'), + 'tweets/search/all': ('GET', 'api'), + 'tweets/search/recent': ('GET', 'api'), + 'tweets/search/stream': ('GET', 'api'), + 'tweets/search/stream/rules': (['POST','GET'], 'api'), + + 'users': ('GET', 'api'), + 'users/:PARAM': ('GET', 'api'), # ID + 'users/:PARAM/blocking': (['GET','POST'], 'api'), # ID + 'users/:PARAM/followed_lists': (['GET','POST'], 'api'), # ID + 'users/:PARAM/followed_lists/:PARAM': ('DELETE', 'api'), # ID, LIST ID + 'users/:PARAM/followers': ('GET', 'api'), # ID + 'users/:PARAM/following': (['GET','POST'], 'api'), # ID + 'users/:PARAM/following/:PARAM': ('DELETE', 'api'), # USER SOURCE ID, USER TARGET ID + 'users/:PARAM/liked_tweets': ('GET', 'api'), # ID + 'users/:PARAM/likes': ('POST', 'api'), # ID + 'users/:PARAM/likes/:PARAM': ('DELETE', 'api'), # USER ID, TWEET ID + 'users/:PARAM/liked_tweets': ('GET', 'api'), # ID + 'users/:PARAM/list_memberships': ('GET', 'api'), # ID + 'users/:PARAM/mentions': ('GET', 'api'), # ID + 'users/:PARAM/muting': ('GET', 'api'), # ID + 'users/:PARAM/owned_lists': ('GET', 'api'), # ID + 'users/:PARAM/pinned_lists': (['GET','POST'], 'api'), # ID + 'users/:PARAM/pinned_lists/:PARAM': ('DELETE', 'api'), # ID, LIST ID + 'users/:PARAM/retweets': ('POST', 'api'), # ID + 'users/:PARAM/retweets/:PARAM': ('DELETE', 'api'), # ID, SOURCE TWEET ID + 'users/:PARAM/tweets': ('GET', 'api'), # ID + 'users/:PARAM/blocking/:PARAM': ('DELETE', 'api'), # SOURCE USER ID, TARGET USER ID + 'users/:PARAM/following/:PARAM': ('DELETE', 'api'), # SOURCE USER ID, TARGET USER ID + 'users/:PARAM/muting/:PARAM': ('DELETE', 'api'), # SOURCE USER ID, TARGET USER ID + 'users/by': ('GET', 'api'), + 'users/by/username/:PARAM': ('GET', 'api'), # USERNAME } From bcc76a3435929551885c2484ae597dee5bd22a75 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Tue, 30 Nov 2021 18:37:45 -0500 Subject: [PATCH 108/139] Added new v2 endpoints --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e974823..719cb9d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ MANIFEST .DS_Store +.vscode/* *.py[cod] From 57fdab13fee416bd7ca370c1f252b6e252ae82a9 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Tue, 14 Dec 2021 11:00:33 -0500 Subject: [PATCH 109/139] Fixed v1.1 POST bug --- CHANGE.log | 2 ++ TwitterAPI/TwitterAPI.py | 11 ++++------- TwitterAPI/__init__.py | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index 7fdeab3..8198aa4 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -192,3 +192,5 @@ v2.7.8, 29 Nov 2021 -- Fixed hydrated media keys v2.7.9, 30 Nov 2021 -- Specify all supported endpoint methods in constants.py Added new v2 endpoints + +v2.7.10, 14 Dec 2021 -- Fixed v1.1 POST bug \ No newline at end of file diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 8d88a42..c31d63c 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -172,13 +172,6 @@ def request(self, resource, params=None, files=None, method_override=None, hydra else: raise Exception(f'Endpoint "{endpoint}" with method "{method_override}" unsupported') - if not params: - params = {} - if self.version == '1.1': - # used only for streaming - params['delimited'] = 'length' - params['stall_warnings'] = 'true' - with requests.Session() as session: session.auth = self.auth session.headers = {'User-Agent': self.USER_AGENT} @@ -186,6 +179,10 @@ def request(self, resource, params=None, files=None, method_override=None, hydra if self.version == '1.1' and 'stream' in subdomain: session.stream = True timeout = self.STREAMING_TIMEOUT + if not params: + params = {} + params['delimited'] = 'length' + params['stall_warnings'] = 'true' elif self.version == '2' and resource.endswith('/stream'): session.stream = True timeout = self.STREAMING_TIMEOUT diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 646f621..16e0609 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.9.1' +__version__ = '2.7.10' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 76fe8eb518b889e61b44e28edd14f15d4f66a645 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Thu, 30 Dec 2021 18:51:26 -0500 Subject: [PATCH 110/139] Fixed bug in conversation_tree.py example --- TwitterAPI/__init__.py | 2 +- examples/v2/conversation_tree.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 16e0609..99c3d8d 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.10' +__version__ = '2.7.10.1' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/examples/v2/conversation_tree.py b/examples/v2/conversation_tree.py index 588d79c..4534632 100644 --- a/examples/v2/conversation_tree.py +++ b/examples/v2/conversation_tree.py @@ -1,4 +1,4 @@ -from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager, HydrateType # NOTE: If conversation is over a week old then it will not get returned. CONVERSATION_ID = '1369393783482236933' @@ -54,7 +54,7 @@ def print_tree(self, level): 'expansions':'author_id', 'tweet.fields':'author_id,conversation_id,created_at,referenced_tweets' }, - hydrate_tweets=True) + hydrate_type=HydrateType.REPLACE) for item in r: root = TreeNode(item) @@ -69,7 +69,7 @@ def print_tree(self, level): 'expansions':'author_id', 'tweet.fields':'author_id,conversation_id,created_at,referenced_tweets' }, - hydrate_tweets=True) + hydrate_type=HydrateType.REPLACE) # "wait=2" means wait 2 seconds between each request. # The rate limit is 450 requests per 15 minutes, or From 70d658ced15b0972fff51d49f8096fd457d14ee2 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sat, 1 Jan 2022 13:49:02 -0500 Subject: [PATCH 111/139] Updated conversation_tree.py example --- TwitterAPI/__init__.py | 2 +- examples/v2/conversation_tree.py | 30 +++++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 99c3d8d..19c6944 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.10.1' +__version__ = '2.7.10.2' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/examples/v2/conversation_tree.py b/examples/v2/conversation_tree.py index 4534632..9ec6b54 100644 --- a/examples/v2/conversation_tree.py +++ b/examples/v2/conversation_tree.py @@ -1,9 +1,19 @@ -from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager, HydrateType +from TwitterAPI import (TwitterAPI, TwitterOAuth, TwitterRequestError, + TwitterConnectionError, TwitterPager, HydrateType) -# NOTE: If conversation is over a week old then it will not get returned. -CONVERSATION_ID = '1369393783482236933' + +# NOTE: If any of the conversation is over a week old then it will not get +# returned unless you are using academic credentials. +CONVERSATION_ID = '20' + + +# +# UTILITY CLASS +# class TreeNode: + """TreeNode is used to organize tweets as a tree structure""" + def __init__(self, data): """data is a tweet's json object""" self.data = data @@ -43,6 +53,11 @@ def print_tree(self, level): for child in reversed(self.children): child.print_tree(level) + +# +# PROGRAM BEGINS HERE +# + try: o = TwitterOAuth.read_file() api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') @@ -56,9 +71,13 @@ def print_tree(self, level): }, hydrate_type=HydrateType.REPLACE) + root = None for item in r: root = TreeNode(item) print(f'ROOT {root.id()}') + if not root: + print(f'Conversation ID {CONVERSATION_ID} does not exist') + exit() # GET ALL REPLIES IN CONVERSATION # (RETURNED IN REVERSE CHRONOLOGICAL ORDER) @@ -72,8 +91,7 @@ def print_tree(self, level): hydrate_type=HydrateType.REPLACE) # "wait=2" means wait 2 seconds between each request. - # The rate limit is 450 requests per 15 minutes, or - # 15*60/450 = 2 seconds. + # The rate limit is 450 requests per 15 minutes, or 1 request every 15*60/450 = 2 seconds. orphans = [] @@ -88,6 +106,8 @@ def print_tree(self, level): print('\nTREE...') root.print_tree(0) + + # YOU MIGHT GET ORPHANS WHEN PART OF THE CONVERSATION IS OLDER THAN A WEEK assert len(orphans) == 0, f'{len(orphans)} orphaned tweets' except TwitterRequestError as e: From 69a0f412ffc2bc8ceb43964784d474abd1e36cd6 Mon Sep 17 00:00:00 2001 From: m Date: Sun, 2 Jan 2022 00:35:27 -0500 Subject: [PATCH 112/139] Historical tweet counts API uses next_token for pagination --- TwitterAPI/TwitterPager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py index e65b0e7..816cde6 100644 --- a/TwitterAPI/TwitterPager.py +++ b/TwitterAPI/TwitterPager.py @@ -123,7 +123,7 @@ def get_iterator(self, wait=5, new_tweets=False): continue else: # VERSION 2 # TWITTER SHOULD STANDARDIZE ON pagination_token IN THE FUTURE - SEARCH_ENDPOINTS = ['tweets/search/recent', 'tweets/search/all'] + SEARCH_ENDPOINTS = ['tweets/search/recent', 'tweets/search/all', 'tweets/counts/all'] pagination_token = 'next_token' if self.resource in SEARCH_ENDPOINTS else 'pagination_token' if new_tweets: self.params[pagination_token] = meta['previous_token'] From 1cb89e8fc50b051707fb99d6c2bb235ada5faf1a Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sun, 2 Jan 2022 11:26:44 -0500 Subject: [PATCH 113/139] Merged m0namon-counts_all_next_token --- CHANGE.log | 4 +++- TwitterAPI/__init__.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index 8198aa4..cfd9353 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -193,4 +193,6 @@ v2.7.8, 29 Nov 2021 -- Fixed hydrated media keys v2.7.9, 30 Nov 2021 -- Specify all supported endpoint methods in constants.py Added new v2 endpoints -v2.7.10, 14 Dec 2021 -- Fixed v1.1 POST bug \ No newline at end of file +v2.7.10, 14 Dec 2021 -- Fixed v1.1 POST bug + +v2.7.11, 02 Jan 2022 -- Merged m0namon-counts_all_next_token diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 19c6944..b448060 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.10.2' +__version__ = '2.7.11' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 324bad7c4afcc94efc5eabcfda4d86b4c9aaebfa Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Wed, 12 Jan 2022 17:28:23 -0500 Subject: [PATCH 114/139] Updated README --- README.rst | 69 ++++++++++++++++++++---------------------- TwitterAPI/__init__.py | 2 +- 2 files changed, 33 insertions(+), 38 deletions(-) diff --git a/README.rst b/README.rst index 24002e8..b20ce15 100755 --- a/README.rst +++ b/README.rst @@ -1,9 +1,9 @@ |LOGO| -============================ +====== |BADGE_VERSION| |BADGE_CHAT| ============================ |BADGE_1.1| |BADGE_PREMIUM| |BADGE_2| |BADGE_ADS| |BADGE_LABS| -============================ +============================================================== .. |LOGO| image:: https://raw.githubusercontent.com/geduldig/TwitterAPI/master/logo.png .. |BADGE_VERSION| image:: http://img.shields.io/pypi/v/TwitterAPI.svg @@ -23,58 +23,53 @@ .. |BADGE_PREMIUM| image:: https://img.shields.io/endpoint?url=https%3A%2F%2Fblue-sea-697d.quartiers047.workers.dev%3A443%2Fhttps%2Ftwbadges.glitch.me%2Fbadges%2Fpremium :target: https://developer.twitter.com -TwitterAPI is a Python package for accessing Twitter's REST APIs and Streaming APIs. It supports both Version 1.1 and Version 2 endpoints. - -REST APIs that are supported are: Public API, Collections API, Curator API, Ads API, Webhook API, Premium Search API. - -NEW -- Twitter API Version 2!! ------------------------------- -By default TwitterAPI will permit only Version 1.1 endpoints. To start using Verson 2 endpoint you must supply the api_version parameter: - - api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret, api_version='2') - -Check the examples folder for more information on making requests using Version 2 endpoints. - Installation ------------ -From the command line:: - pip install TwitterAPI + > pip install TwitterAPI -Some Code... ------------- -[See `TwitterAPI/examples `_ for working examples.] +Twitter API Version 1.1 Code Snippets +------------------------------------- +[More examples in `TwitterAPI/examples/v1.1 `_] -First, authenticate with your application credentials:: +Search for recent tweets +:: from TwitterAPI import TwitterAPI api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret) + r = api.request('search/tweets', {'q':'pizza'}) + for item in r: + print(item) -Tweet something:: - - r = api.request('statuses/update', {'status':'This is a tweet!'}) - print(r.status_code) +Stream tweets from New York City as they get tweeted +:: -Get tweet by its id:: + r = api.request('statuses/filter', {'locations':'-74,40,-73,41'}) + for item in r: + print(item) - r = api.request('statuses/show/:%d' % 210462857140252672) - print(r.text) +Twitter API Version 2 Code Snippets +------------------------------------ +[More examples in `TwitterAPI/examples/v2 `_ ] -Get some tweets:: +Search for recent tweets, and specify `fields` and `expansions` +:: - r = api.request('search/tweets', {'q':'pizza'}) + from TwitterAPI import TwitterAPI + api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret, api_version='2') + r = api.request('tweets/search/recent', { + 'query':'pizza', + 'tweet.fields':'author_id', + 'expansions':'author_id'}) for item in r: print(item) -Stream tweets from New York City:: +The ``request()`` Method +------------------------ - r = api.request('statuses/filter', {'locations':'-74,40,-73,41'}) - for item in r: - print(item) - -``request()`` works with all version 1.1 and version 2 endpoints and the other supported Twitter APIs. Typcally, ``request()`` takes two arguments: a Twitter endpoint and a dictionary of endpoint parameters. The above examples use ``get_iterator()`` to consume each tweet object. The iterator can iterate results returned from either the REST APIs or the Streaming APIs. +This method works with all version 1.1 and version 2 endpoints. Typcally, ``request()`` takes two arguments: a Twitter endpoint and a dictionary of endpoint parameters. -You also have access to the response object returned by ``request()``. With a response object ``r`` comes the raw response (``r.text``) and the HTTP status code (``r.status_code``). See the `requests `_ library documentation for more details. +The method returns an object that will iterate either search results and streams. The returned object also give you access to the raw response (``r.text``) and the HTTP status code (``r.status_code``). See the `requests `_ library documentation for more details. Documentation ------------- @@ -87,4 +82,4 @@ Documentation Extra Goodies ------------- -Command-Line Utility (`examples/cli `_) +Command-Line Utility (`examples/cli `_) \ No newline at end of file diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index b448060..71eca6d 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.11' +__version__ = '2.7.11.1' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 8e70d44e8766be8997656e8d01137562e6a2e07e Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Wed, 12 Jan 2022 17:33:30 -0500 Subject: [PATCH 115/139] Updated README --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index b20ce15..17bc443 100755 --- a/README.rst +++ b/README.rst @@ -1,9 +1,9 @@ |LOGO| ====== -|BADGE_VERSION| |BADGE_CHAT| -============================ |BADGE_1.1| |BADGE_PREMIUM| |BADGE_2| |BADGE_ADS| |BADGE_LABS| ============================================================== +|BADGE_VERSION| |BADGE_CHAT| +============================ .. |LOGO| image:: https://raw.githubusercontent.com/geduldig/TwitterAPI/master/logo.png .. |BADGE_VERSION| image:: http://img.shields.io/pypi/v/TwitterAPI.svg @@ -69,7 +69,7 @@ The ``request()`` Method This method works with all version 1.1 and version 2 endpoints. Typcally, ``request()`` takes two arguments: a Twitter endpoint and a dictionary of endpoint parameters. -The method returns an object that will iterate either search results and streams. The returned object also give you access to the raw response (``r.text``) and the HTTP status code (``r.status_code``). See the `requests `_ library documentation for more details. +The method returns an object that will iterate either search results and streams. The returned object also gives you access to the raw response (``r.text``) and the HTTP status code (``r.status_code``). See the `requests `_ library documentation for more details. Documentation ------------- @@ -82,4 +82,4 @@ Documentation Extra Goodies ------------- -Command-Line Utility (`examples/cli `_) \ No newline at end of file +Command-Line Utility (`examples/cli `_) From c82895bb83e0dd2745f804c8cd260ec73a3ade3e Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Wed, 12 Jan 2022 17:35:42 -0500 Subject: [PATCH 116/139] Updated README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 17bc443..bce25f2 100755 --- a/README.rst +++ b/README.rst @@ -33,7 +33,7 @@ Twitter API Version 1.1 Code Snippets [More examples in `TwitterAPI/examples/v1.1 `_] Search for recent tweets -:: +.. code-block:: python from TwitterAPI import TwitterAPI api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret) From c49ca38ac4ae1f3879ba0f84670955bdaff6679c Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Wed, 12 Jan 2022 17:37:36 -0500 Subject: [PATCH 117/139] Updated README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index bce25f2..0f84465 100755 --- a/README.rst +++ b/README.rst @@ -33,7 +33,7 @@ Twitter API Version 1.1 Code Snippets [More examples in `TwitterAPI/examples/v1.1 `_] Search for recent tweets -.. code-block:: python +.. code:: python from TwitterAPI import TwitterAPI api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret) From ad393e0cb57ba840f41cbefbcc4aa7795e229461 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Wed, 12 Jan 2022 17:39:47 -0500 Subject: [PATCH 118/139] Updated README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 0f84465..17bc443 100755 --- a/README.rst +++ b/README.rst @@ -33,7 +33,7 @@ Twitter API Version 1.1 Code Snippets [More examples in `TwitterAPI/examples/v1.1 `_] Search for recent tweets -.. code:: python +:: from TwitterAPI import TwitterAPI api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret) From 24fd6ee2245649d5c3e571ce47eff6b4e6ee6fa5 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Wed, 12 Jan 2022 17:41:00 -0500 Subject: [PATCH 119/139] Updated README --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 17bc443..5b9112e 100755 --- a/README.rst +++ b/README.rst @@ -28,9 +28,9 @@ Installation > pip install TwitterAPI -Twitter API Version 1.1 Code Snippets +Twitter API Version 1.1 Code Snippets [More examples in `TwitterAPI/examples/v1.1 `_] ------------------------------------- -[More examples in `TwitterAPI/examples/v1.1 `_] + Search for recent tweets :: From 1e76fef20aaf71d37be27b323cff3aa6ba3fee10 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Wed, 12 Jan 2022 17:41:34 -0500 Subject: [PATCH 120/139] Updated README --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 5b9112e..b9ab0c6 100755 --- a/README.rst +++ b/README.rst @@ -28,9 +28,9 @@ Installation > pip install TwitterAPI -Twitter API Version 1.1 Code Snippets [More examples in `TwitterAPI/examples/v1.1 `_] +Twitter API Version 1.1 Code Snippets ------------------------------------- - +[More examples in `TwitterAPI/examples/v1.1 `_] Search for recent tweets :: From 4491faf4f26f1e46385fefb7d085e0192d24ab59 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Thu, 13 Jan 2022 13:47:23 -0500 Subject: [PATCH 121/139] Updated README --- README.rst | 20 +++++++++++++------- TwitterAPI/__init__.py | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index b9ab0c6..34a45d5 100755 --- a/README.rst +++ b/README.rst @@ -1,9 +1,5 @@ |LOGO| ====== -|BADGE_1.1| |BADGE_PREMIUM| |BADGE_2| |BADGE_ADS| |BADGE_LABS| -============================================================== -|BADGE_VERSION| |BADGE_CHAT| -============================ .. |LOGO| image:: https://raw.githubusercontent.com/geduldig/TwitterAPI/master/logo.png .. |BADGE_VERSION| image:: http://img.shields.io/pypi/v/TwitterAPI.svg @@ -23,6 +19,16 @@ .. |BADGE_PREMIUM| image:: https://img.shields.io/endpoint?url=https%3A%2F%2Fblue-sea-697d.quartiers047.workers.dev%3A443%2Fhttps%2Ftwbadges.glitch.me%2Fbadges%2Fpremium :target: https://developer.twitter.com + +TwitterAPI is a minimal python wrapper for the TwitterAPIs. A list of what it can do: + +* Support for all V1.1 and V2 endpoints, plus Premium, Ads, Labs, Collections. +* OAuth1 and bearer token authentication, and proxy server authentication. +* Streaming endpoints. +* Paging results. +* The option to "hydrate" results returned by V2 endpoints. +* Error handling. + Installation ------------ @@ -64,10 +70,10 @@ Search for recent tweets, and specify `fields` and `expansions` for item in r: print(item) -The ``request()`` Method ------------------------- +One Method For Everything +------------------------- -This method works with all version 1.1 and version 2 endpoints. Typcally, ``request()`` takes two arguments: a Twitter endpoint and a dictionary of endpoint parameters. +The ``request()`` method works with all version 1.1 and version 2 endpoints. Typcally, ``request()`` takes two arguments: a Twitter endpoint and a dictionary of endpoint parameters. The method returns an object that will iterate either search results and streams. The returned object also gives you access to the raw response (``r.text``) and the HTTP status code (``r.status_code``). See the `requests `_ library documentation for more details. diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 71eca6d..bfe11b2 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.11.1' +__version__ = '2.7.11.2' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 4985bf5e885cacc4237ecf9a0a0a0a9a44467d65 Mon Sep 17 00:00:00 2001 From: Justin Nebo Date: Thu, 10 Feb 2022 20:26:38 +0100 Subject: [PATCH 122/139] update ads api to version 10 --- TwitterAPI/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index adb65f2..fd3cbf0 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -20,7 +20,7 @@ VERSION = '1.1' CURATOR_VERSION = 'broadcast/1' -ADS_VERSION = '9' +ADS_VERSION = '10' ENDPOINTS = { From 894b5020c78b9021c76ca00a80ce9ed046190d1e Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Fri, 11 Feb 2022 09:39:08 -0500 Subject: [PATCH 123/139] merged Dheavyman-update-ads-api-v10 --- CHANGE.log | 2 ++ TwitterAPI/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGE.log b/CHANGE.log index cfd9353..a667cac 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -196,3 +196,5 @@ v2.7.9, 30 Nov 2021 -- Specify all supported endpoint methods in constants.py v2.7.10, 14 Dec 2021 -- Fixed v1.1 POST bug v2.7.11, 02 Jan 2022 -- Merged m0namon-counts_all_next_token + +v2.7.12, 11 Feb 2022 -- Merged Dheavyman-update-ads-api-v10 diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index bfe11b2..ba65d81 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.11.2' +__version__ = '2.7.12' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 1c1a61aa6bc84b65b3f9c7900b57209cd5f66334 Mon Sep 17 00:00:00 2001 From: hamzsait Date: Thu, 24 Feb 2022 19:34:28 -0600 Subject: [PATCH 124/139] added lookup tweets.py --- examples/v2/lookup_tweets.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 examples/v2/lookup_tweets.py diff --git a/examples/v2/lookup_tweets.py b/examples/v2/lookup_tweets.py new file mode 100644 index 0000000..aa1e370 --- /dev/null +++ b/examples/v2/lookup_tweets.py @@ -0,0 +1,24 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError + +TWEET_IDS = ['964575983633252353', '1497014169178185730'] + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets',{'ids':",".join(TWEET_IDS)}) + + for item in r: + print(item) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file From b2e28bc4ea4ad136c53175fab955ae3923f757b6 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 6 Jun 2022 09:13:48 -0400 Subject: [PATCH 125/139] fixed conflict in constants.py --- CHANGE.log | 2 ++ TwitterAPI/__init__.py | 2 +- TwitterAPI/constants.py | 47 ++++++++++++++++++++--------------------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index a667cac..2e32139 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -198,3 +198,5 @@ v2.7.10, 14 Dec 2021 -- Fixed v1.1 POST bug v2.7.11, 02 Jan 2022 -- Merged m0namon-counts_all_next_token v2.7.12, 11 Feb 2022 -- Merged Dheavyman-update-ads-api-v10 + +v2.7.13, 06 Jun 2022 -- Fixed endpoint conflict in constanst.py \ No newline at end of file diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index ba65d81..e5ab050 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.12' +__version__ = '2.7.13' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index fd3cbf0..3140832 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -206,36 +206,35 @@ # ACCOUNT ACTIVITY WEBHOOK API - 'account_activity/all/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/subscriptions': ('DELETE', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/subscriptions/all': ('GET', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/subscriptions/all/list': ('GET', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/webhooks': ('POST', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/webhooks/:PARAM': ('DELETE', 'api'), # ENVIRONMENT NAME, WEBHOOK ID - 'account_activity/all/count': ('GET', 'api'), - 'account_activity/all/webhooks': ('GET', 'api'), - 'account_activity/webhooks': ('POST', 'api'), - 'account_activity/webhooks/:PARAM': ('DELETE', 'api'), # WEBHOOK ID - 'account_activity/webhooks/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME - 'account_activity/webhooks/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions': (['POST', 'DELETE'], 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions/all': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions/all/list': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/webhooks': ('POST', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/webhooks/:PARAM': ('DELETE', 'api'), # ENVIRONMENT NAME, WEBHOOK ID + 'account_activity/all/count': ('GET', 'api'), + 'account_activity/all/webhooks': ('GET', 'api'), + 'account_activity/webhooks': ('POST', 'api'), + 'account_activity/webhooks/:PARAM': ('DELETE', 'api'), # WEBHOOK ID + 'account_activity/webhooks/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME + 'account_activity/webhooks/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME # PREMIUM SEARCH API - 'tweets/search/30day/:PARAM': ('GET', 'api'), # LABEL - 'tweets/search/30day/:PARAM/counts': ('GET', 'api'), # LABEL - 'tweets/search/fullarchive/:PARAM': ('GET', 'api'), # LABEL - 'tweets/search/fullarchive/:PARAM/counts': ('GET', 'api'), # LABEL + 'tweets/search/30day/:PARAM': ('GET', 'api'), # LABEL + 'tweets/search/30day/:PARAM/counts': ('GET', 'api'), # LABEL + 'tweets/search/fullarchive/:PARAM': ('GET', 'api'), # LABEL + 'tweets/search/fullarchive/:PARAM/counts': ('GET', 'api'), # LABEL # LABS API (BETAS) WILL NEED APPLICATION APPROVAL - 'labs/1/tweets/metrics/private': ('GET', 'api'), - 'labs/2/tweets/:PARAM': ('GET', 'api'), # TWEET ID - 'labs/2/tweets': ('GET', 'api'), - 'labs/2/tweets/search': ('GET', 'api'), - 'labs/2/tweets/:PARAM/hidden': ('PUT', 'api'), # TWEET ID - 'labs/2/users/:PARAM': ('GET', 'api'), # USER ID - 'labs/2/users': ('GET', 'api'), + 'labs/1/tweets/metrics/private': ('GET', 'api'), + 'labs/2/tweets/:PARAM': ('GET', 'api'), # TWEET ID + 'labs/2/tweets': ('GET', 'api'), + 'labs/2/tweets/search': ('GET', 'api'), + 'labs/2/tweets/:PARAM/hidden': ('PUT', 'api'), # TWEET ID + 'labs/2/users/:PARAM': ('GET', 'api'), # USER ID + 'labs/2/users': ('GET', 'api'), # !!!!!!!!!!!!!!!!!!! # API V2 EARLY ACCESS From e0fd46ca3ce5af599087448cc57c3ef5873e76d7 Mon Sep 17 00:00:00 2001 From: Sebastian Steins Date: Thu, 1 Sep 2022 12:52:30 +0200 Subject: [PATCH 126/139] First test with bookmarks and known OAuth2 token --- TwitterAPI/BearerAuthUser.py | 64 +++++++++++++++++++++++++++++++++++ TwitterAPI/TwitterAPI.py | 12 +++++++ TwitterAPI/constants.py | 1 + examples/v2/user_bookmarks.py | 44 ++++++++++++++++++++++++ 4 files changed, 121 insertions(+) create mode 100644 TwitterAPI/BearerAuthUser.py create mode 100644 examples/v2/user_bookmarks.py diff --git a/TwitterAPI/BearerAuthUser.py b/TwitterAPI/BearerAuthUser.py new file mode 100644 index 0000000..1482cf2 --- /dev/null +++ b/TwitterAPI/BearerAuthUser.py @@ -0,0 +1,64 @@ +__author__ = "Andrea Biancini, geduldig" +__date__ = "January 3, 2014" +__license__ = "MIT" + + +from .constants import * +import base64 +import requests + + +OAUTH2_SUBDOMAIN = 'api' +OAUTH2_ENDPOINT = 'oauth2/token' + + +class BearerAuthUser(requests.auth.AuthBase): + + """Request bearer access token for oAuth2 authentication. + + :param consumer_key: Twitter application consumer key + :param consumer_secret: Twitter application consumer secret + :param proxies: Dictionary of proxy URLs (see documentation for python-requests). + """ + + def __init__(self, consumer_key, consumer_secret, oauth2_access_token, proxies=None, user_agent=None): + self._consumer_key = consumer_key + self._consumer_secret = consumer_secret + self._access_token = oauth2_access_token + self.proxies = proxies + self.user_agent = user_agent + self._bearer_token = self._get_access_token() + + def _get_access_token(self): + token_url = '%s://%s.%s/%s' % (PROTOCOL, + OAUTH2_SUBDOMAIN, + DOMAIN, + OAUTH2_ENDPOINT) + auth = self._consumer_key + ':' + self._consumer_secret + b64_bearer_token_creds = base64.b64encode(auth.encode('utf8')) + params = {'grant_type': 'client_credentials'} + headers = {} + headers['User-Agent'] = self.user_agent + headers['Authorization'] = 'Basic ' + b64_bearer_token_creds.decode('utf8') + headers['Content-Type'] = 'application/x-www-form-urlencoded;charset=UTF-8' + try: + response = requests.post( + token_url, + params=params, + headers=headers, + proxies=self.proxies) + data = response.json() + return data['access_token'] + except Exception as e: + raise Exception('Error requesting bearer access token: %s' % e) + + def __call__(self, r): + auth_list = [ + self._consumer_key, + self._consumer_secret, + self._access_token] + if all(auth_list): + r.headers['Authorization'] = "Bearer %s" % self._access_token + return r + else: + raise Exception('Not enough keys passed to Bearer token manager.') diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index c31d63c..5656fce 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -4,6 +4,7 @@ from .BearerAuth import BearerAuth as OAuth2 +from .BearerAuthUser import BearerAuthUser as OAuth2User from .constants import * from .TwitterError import * from datetime import datetime @@ -28,6 +29,7 @@ class OAuthType(Enum): OAUTH1 = 'oAuth1' OAUTH2 = 'oAuth2' + OAUTH2USER = 'oAuth2User' class HydrateType(Enum): @@ -61,6 +63,7 @@ def __init__( consumer_secret=None, access_token_key=None, access_token_secret=None, + oauth2_access_token=None, auth_type=OAuthType.OAUTH1, proxy_url=None, api_version=VERSION): @@ -94,6 +97,15 @@ def __init__( consumer_secret, proxies=self.proxies, user_agent=self.USER_AGENT) + elif auth_type == OAuthType.OAUTH2USER or auth_type == 'oAuth2User': + if not all([consumer_key, consumer_secret, oauth2_access_token]): + raise Exception('Missing authentication parameter') + self.auth = OAuth2User( + consumer_key, + consumer_secret, + oauth2_access_token=oauth2_access_token, + proxies=self.proxies, + user_agent=self.USER_AGENT) else: raise Exception('Unknown oAuth version') diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 3140832..2d26115 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -272,6 +272,7 @@ 'users': ('GET', 'api'), 'users/:PARAM': ('GET', 'api'), # ID 'users/:PARAM/blocking': (['GET','POST'], 'api'), # ID + 'users/:PARAM/bookmarks': (['GET','DELETE'], 'api'), # ID 'users/:PARAM/followed_lists': (['GET','POST'], 'api'), # ID 'users/:PARAM/followed_lists/:PARAM': ('DELETE', 'api'), # ID, LIST ID 'users/:PARAM/followers': ('GET', 'api'), # ID diff --git a/examples/v2/user_bookmarks.py b/examples/v2/user_bookmarks.py new file mode 100644 index 0000000..a8ffc09 --- /dev/null +++ b/examples/v2/user_bookmarks.py @@ -0,0 +1,44 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +from pprint import pprint + +USER_ID = '1249069835562233858' # https://twitter.com/bascodes +access_token = "" + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, oauth2_access_token=access_token, api_version='2', auth_type="oAuth2User") + + # Get tweets - default setting + tweets = api.request(f'users/:{USER_ID}/bookmarks') + for t in tweets: + print(t) + + # Get tweets with customization - (5 tweets only with created_at timestamp) + print() + params = {'max_results': 5, 'tweet.fields': 'created_at'} + tweets = api.request(f'users/:{USER_ID}/bookmarks', params) + for t in tweets: + pprint(t) + + # Get next 5 tweets + print() + next_token = tweets.json()['meta']['next_token'] + params = {'max_results': 5, 'tweet.fields': 'created_at', 'pagination_token': next_token} + tweets = api.request(f'users/:{USER_ID}/bookmarks', params) + for t in tweets: + pprint(t) + + +except TwitterRequestError as e: + print('Request error') + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print('Connection error') + print(e) + +except Exception as e: + print('Exception') + print(e) \ No newline at end of file From 48f2f81bb727797c780952cb407d259508e1e68b Mon Sep 17 00:00:00 2001 From: Sebastian Steins Date: Thu, 1 Sep 2022 12:59:37 +0200 Subject: [PATCH 127/139] Add support for known OAuth2 token to enable user-context requests Closes #219 Closes #220 --- TwitterAPI/BearerAuthUser.py | 40 ++++------------------------------- TwitterAPI/TwitterAPI.py | 4 +--- examples/v2/user_bookmarks.py | 2 +- 3 files changed, 6 insertions(+), 40 deletions(-) diff --git a/TwitterAPI/BearerAuthUser.py b/TwitterAPI/BearerAuthUser.py index 1482cf2..8f932c8 100644 --- a/TwitterAPI/BearerAuthUser.py +++ b/TwitterAPI/BearerAuthUser.py @@ -4,14 +4,9 @@ from .constants import * -import base64 import requests -OAUTH2_SUBDOMAIN = 'api' -OAUTH2_ENDPOINT = 'oauth2/token' - - class BearerAuthUser(requests.auth.AuthBase): """Request bearer access token for oAuth2 authentication. @@ -21,44 +16,17 @@ class BearerAuthUser(requests.auth.AuthBase): :param proxies: Dictionary of proxy URLs (see documentation for python-requests). """ - def __init__(self, consumer_key, consumer_secret, oauth2_access_token, proxies=None, user_agent=None): - self._consumer_key = consumer_key - self._consumer_secret = consumer_secret - self._access_token = oauth2_access_token + def __init__(self, oauth2_access_token, proxies=None, user_agent=None): self.proxies = proxies self.user_agent = user_agent - self._bearer_token = self._get_access_token() + self._bearer_token = oauth2_access_token - def _get_access_token(self): - token_url = '%s://%s.%s/%s' % (PROTOCOL, - OAUTH2_SUBDOMAIN, - DOMAIN, - OAUTH2_ENDPOINT) - auth = self._consumer_key + ':' + self._consumer_secret - b64_bearer_token_creds = base64.b64encode(auth.encode('utf8')) - params = {'grant_type': 'client_credentials'} - headers = {} - headers['User-Agent'] = self.user_agent - headers['Authorization'] = 'Basic ' + b64_bearer_token_creds.decode('utf8') - headers['Content-Type'] = 'application/x-www-form-urlencoded;charset=UTF-8' - try: - response = requests.post( - token_url, - params=params, - headers=headers, - proxies=self.proxies) - data = response.json() - return data['access_token'] - except Exception as e: - raise Exception('Error requesting bearer access token: %s' % e) def __call__(self, r): auth_list = [ - self._consumer_key, - self._consumer_secret, - self._access_token] + self._bearer_token] if all(auth_list): - r.headers['Authorization'] = "Bearer %s" % self._access_token + r.headers['Authorization'] = "Bearer %s" % self._bearer_token return r else: raise Exception('Not enough keys passed to Bearer token manager.') diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 5656fce..f14d879 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -98,11 +98,9 @@ def __init__( proxies=self.proxies, user_agent=self.USER_AGENT) elif auth_type == OAuthType.OAUTH2USER or auth_type == 'oAuth2User': - if not all([consumer_key, consumer_secret, oauth2_access_token]): + if not all([oauth2_access_token]): raise Exception('Missing authentication parameter') self.auth = OAuth2User( - consumer_key, - consumer_secret, oauth2_access_token=oauth2_access_token, proxies=self.proxies, user_agent=self.USER_AGENT) diff --git a/examples/v2/user_bookmarks.py b/examples/v2/user_bookmarks.py index a8ffc09..87cbbe1 100644 --- a/examples/v2/user_bookmarks.py +++ b/examples/v2/user_bookmarks.py @@ -6,7 +6,7 @@ try: o = TwitterOAuth.read_file() - api = TwitterAPI(o.consumer_key, o.consumer_secret, oauth2_access_token=access_token, api_version='2', auth_type="oAuth2User") + api = TwitterAPI(oauth2_access_token=access_token, api_version='2', auth_type="oAuth2User") # Get tweets - default setting tweets = api.request(f'users/:{USER_ID}/bookmarks') From 951edb62fb75a2393e95447e229090b330339f4f Mon Sep 17 00:00:00 2001 From: Justin Nebo Date: Tue, 13 Sep 2022 20:31:05 +0100 Subject: [PATCH 128/139] update twitter ads api from v10 to v11 --- TwitterAPI/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 3140832..3e3ff57 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -20,7 +20,7 @@ VERSION = '1.1' CURATOR_VERSION = 'broadcast/1' -ADS_VERSION = '10' +ADS_VERSION = '11' ENDPOINTS = { From f48b14fb7b756eafa31efde1cbb2b21aa441870b Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 6 Jun 2022 09:13:48 -0400 Subject: [PATCH 129/139] fixed conflict in constants.py --- CHANGE.log | 2 ++ TwitterAPI/__init__.py | 2 +- TwitterAPI/constants.py | 47 ++++++++++++++++++++--------------------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index a667cac..2e32139 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -198,3 +198,5 @@ v2.7.10, 14 Dec 2021 -- Fixed v1.1 POST bug v2.7.11, 02 Jan 2022 -- Merged m0namon-counts_all_next_token v2.7.12, 11 Feb 2022 -- Merged Dheavyman-update-ads-api-v10 + +v2.7.13, 06 Jun 2022 -- Fixed endpoint conflict in constanst.py \ No newline at end of file diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index ba65d81..e5ab050 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.12' +__version__ = '2.7.13' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index fd3cbf0..3140832 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -206,36 +206,35 @@ # ACCOUNT ACTIVITY WEBHOOK API - 'account_activity/all/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/subscriptions': ('DELETE', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/subscriptions/all': ('GET', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/subscriptions/all/list': ('GET', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/webhooks': ('POST', 'api'), # ENVIRONMENT NAME - 'account_activity/all/:PARAM/webhooks/:PARAM': ('DELETE', 'api'), # ENVIRONMENT NAME, WEBHOOK ID - 'account_activity/all/count': ('GET', 'api'), - 'account_activity/all/webhooks': ('GET', 'api'), - 'account_activity/webhooks': ('POST', 'api'), - 'account_activity/webhooks/:PARAM': ('DELETE', 'api'), # WEBHOOK ID - 'account_activity/webhooks/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME - 'account_activity/webhooks/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions': (['POST', 'DELETE'], 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions/all': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions/all/list': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/webhooks': ('POST', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/webhooks/:PARAM': ('DELETE', 'api'), # ENVIRONMENT NAME, WEBHOOK ID + 'account_activity/all/count': ('GET', 'api'), + 'account_activity/all/webhooks': ('GET', 'api'), + 'account_activity/webhooks': ('POST', 'api'), + 'account_activity/webhooks/:PARAM': ('DELETE', 'api'), # WEBHOOK ID + 'account_activity/webhooks/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME + 'account_activity/webhooks/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME # PREMIUM SEARCH API - 'tweets/search/30day/:PARAM': ('GET', 'api'), # LABEL - 'tweets/search/30day/:PARAM/counts': ('GET', 'api'), # LABEL - 'tweets/search/fullarchive/:PARAM': ('GET', 'api'), # LABEL - 'tweets/search/fullarchive/:PARAM/counts': ('GET', 'api'), # LABEL + 'tweets/search/30day/:PARAM': ('GET', 'api'), # LABEL + 'tweets/search/30day/:PARAM/counts': ('GET', 'api'), # LABEL + 'tweets/search/fullarchive/:PARAM': ('GET', 'api'), # LABEL + 'tweets/search/fullarchive/:PARAM/counts': ('GET', 'api'), # LABEL # LABS API (BETAS) WILL NEED APPLICATION APPROVAL - 'labs/1/tweets/metrics/private': ('GET', 'api'), - 'labs/2/tweets/:PARAM': ('GET', 'api'), # TWEET ID - 'labs/2/tweets': ('GET', 'api'), - 'labs/2/tweets/search': ('GET', 'api'), - 'labs/2/tweets/:PARAM/hidden': ('PUT', 'api'), # TWEET ID - 'labs/2/users/:PARAM': ('GET', 'api'), # USER ID - 'labs/2/users': ('GET', 'api'), + 'labs/1/tweets/metrics/private': ('GET', 'api'), + 'labs/2/tweets/:PARAM': ('GET', 'api'), # TWEET ID + 'labs/2/tweets': ('GET', 'api'), + 'labs/2/tweets/search': ('GET', 'api'), + 'labs/2/tweets/:PARAM/hidden': ('PUT', 'api'), # TWEET ID + 'labs/2/users/:PARAM': ('GET', 'api'), # USER ID + 'labs/2/users': ('GET', 'api'), # !!!!!!!!!!!!!!!!!!! # API V2 EARLY ACCESS From a26cea507eee14c86a43ee35f954963bded18349 Mon Sep 17 00:00:00 2001 From: hamzsait Date: Thu, 24 Feb 2022 19:34:28 -0600 Subject: [PATCH 130/139] added lookup tweets.py --- examples/v2/lookup_tweets.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 examples/v2/lookup_tweets.py diff --git a/examples/v2/lookup_tweets.py b/examples/v2/lookup_tweets.py new file mode 100644 index 0000000..aa1e370 --- /dev/null +++ b/examples/v2/lookup_tweets.py @@ -0,0 +1,24 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError + +TWEET_IDS = ['964575983633252353', '1497014169178185730'] + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets',{'ids':",".join(TWEET_IDS)}) + + for item in r: + print(item) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file From 5e33153c1a7f3cc5012af4ab4ef22b7bd39f6440 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sun, 16 Oct 2022 22:42:22 -0400 Subject: [PATCH 131/139] Merged codewithbas-feature/user-bookmarks --- CHANGE.log | 4 +++- TwitterAPI/__init__.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index 2e32139..626d0a4 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -199,4 +199,6 @@ v2.7.11, 02 Jan 2022 -- Merged m0namon-counts_all_next_token v2.7.12, 11 Feb 2022 -- Merged Dheavyman-update-ads-api-v10 -v2.7.13, 06 Jun 2022 -- Fixed endpoint conflict in constanst.py \ No newline at end of file +v2.7.13, 06 Jun 2022 -- Fixed endpoint conflict in constanst.py + +v2.8.0, 16 Oct 2022 -- Merged codewithbas-feature/user-bookmarks \ No newline at end of file diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index e5ab050..d5c2520 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.13' +__version__ = '2.8.0' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From ba95d1ae3a67d11a33aff0bdf28df849552ed172 Mon Sep 17 00:00:00 2001 From: hamzsait Date: Thu, 24 Feb 2022 19:34:28 -0600 Subject: [PATCH 132/139] added lookup tweets.py --- examples/v2/lookup_tweets.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 examples/v2/lookup_tweets.py diff --git a/examples/v2/lookup_tweets.py b/examples/v2/lookup_tweets.py new file mode 100644 index 0000000..aa1e370 --- /dev/null +++ b/examples/v2/lookup_tweets.py @@ -0,0 +1,24 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError + +TWEET_IDS = ['964575983633252353', '1497014169178185730'] + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets',{'ids':",".join(TWEET_IDS)}) + + for item in r: + print(item) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file From a15bd7686cd541b61d72906028b57b7c481249ea Mon Sep 17 00:00:00 2001 From: Sebastian Steins Date: Thu, 1 Sep 2022 12:52:30 +0200 Subject: [PATCH 133/139] First test with bookmarks and known OAuth2 token --- TwitterAPI/BearerAuthUser.py | 64 +++++++++++++++++++++++++++++++++++ TwitterAPI/TwitterAPI.py | 12 +++++++ TwitterAPI/constants.py | 1 + examples/v2/user_bookmarks.py | 44 ++++++++++++++++++++++++ 4 files changed, 121 insertions(+) create mode 100644 TwitterAPI/BearerAuthUser.py create mode 100644 examples/v2/user_bookmarks.py diff --git a/TwitterAPI/BearerAuthUser.py b/TwitterAPI/BearerAuthUser.py new file mode 100644 index 0000000..1482cf2 --- /dev/null +++ b/TwitterAPI/BearerAuthUser.py @@ -0,0 +1,64 @@ +__author__ = "Andrea Biancini, geduldig" +__date__ = "January 3, 2014" +__license__ = "MIT" + + +from .constants import * +import base64 +import requests + + +OAUTH2_SUBDOMAIN = 'api' +OAUTH2_ENDPOINT = 'oauth2/token' + + +class BearerAuthUser(requests.auth.AuthBase): + + """Request bearer access token for oAuth2 authentication. + + :param consumer_key: Twitter application consumer key + :param consumer_secret: Twitter application consumer secret + :param proxies: Dictionary of proxy URLs (see documentation for python-requests). + """ + + def __init__(self, consumer_key, consumer_secret, oauth2_access_token, proxies=None, user_agent=None): + self._consumer_key = consumer_key + self._consumer_secret = consumer_secret + self._access_token = oauth2_access_token + self.proxies = proxies + self.user_agent = user_agent + self._bearer_token = self._get_access_token() + + def _get_access_token(self): + token_url = '%s://%s.%s/%s' % (PROTOCOL, + OAUTH2_SUBDOMAIN, + DOMAIN, + OAUTH2_ENDPOINT) + auth = self._consumer_key + ':' + self._consumer_secret + b64_bearer_token_creds = base64.b64encode(auth.encode('utf8')) + params = {'grant_type': 'client_credentials'} + headers = {} + headers['User-Agent'] = self.user_agent + headers['Authorization'] = 'Basic ' + b64_bearer_token_creds.decode('utf8') + headers['Content-Type'] = 'application/x-www-form-urlencoded;charset=UTF-8' + try: + response = requests.post( + token_url, + params=params, + headers=headers, + proxies=self.proxies) + data = response.json() + return data['access_token'] + except Exception as e: + raise Exception('Error requesting bearer access token: %s' % e) + + def __call__(self, r): + auth_list = [ + self._consumer_key, + self._consumer_secret, + self._access_token] + if all(auth_list): + r.headers['Authorization'] = "Bearer %s" % self._access_token + return r + else: + raise Exception('Not enough keys passed to Bearer token manager.') diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index c31d63c..5656fce 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -4,6 +4,7 @@ from .BearerAuth import BearerAuth as OAuth2 +from .BearerAuthUser import BearerAuthUser as OAuth2User from .constants import * from .TwitterError import * from datetime import datetime @@ -28,6 +29,7 @@ class OAuthType(Enum): OAUTH1 = 'oAuth1' OAUTH2 = 'oAuth2' + OAUTH2USER = 'oAuth2User' class HydrateType(Enum): @@ -61,6 +63,7 @@ def __init__( consumer_secret=None, access_token_key=None, access_token_secret=None, + oauth2_access_token=None, auth_type=OAuthType.OAUTH1, proxy_url=None, api_version=VERSION): @@ -94,6 +97,15 @@ def __init__( consumer_secret, proxies=self.proxies, user_agent=self.USER_AGENT) + elif auth_type == OAuthType.OAUTH2USER or auth_type == 'oAuth2User': + if not all([consumer_key, consumer_secret, oauth2_access_token]): + raise Exception('Missing authentication parameter') + self.auth = OAuth2User( + consumer_key, + consumer_secret, + oauth2_access_token=oauth2_access_token, + proxies=self.proxies, + user_agent=self.USER_AGENT) else: raise Exception('Unknown oAuth version') diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 3e3ff57..5c2c864 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -272,6 +272,7 @@ 'users': ('GET', 'api'), 'users/:PARAM': ('GET', 'api'), # ID 'users/:PARAM/blocking': (['GET','POST'], 'api'), # ID + 'users/:PARAM/bookmarks': (['GET','DELETE'], 'api'), # ID 'users/:PARAM/followed_lists': (['GET','POST'], 'api'), # ID 'users/:PARAM/followed_lists/:PARAM': ('DELETE', 'api'), # ID, LIST ID 'users/:PARAM/followers': ('GET', 'api'), # ID diff --git a/examples/v2/user_bookmarks.py b/examples/v2/user_bookmarks.py new file mode 100644 index 0000000..a8ffc09 --- /dev/null +++ b/examples/v2/user_bookmarks.py @@ -0,0 +1,44 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +from pprint import pprint + +USER_ID = '1249069835562233858' # https://twitter.com/bascodes +access_token = "" + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, oauth2_access_token=access_token, api_version='2', auth_type="oAuth2User") + + # Get tweets - default setting + tweets = api.request(f'users/:{USER_ID}/bookmarks') + for t in tweets: + print(t) + + # Get tweets with customization - (5 tweets only with created_at timestamp) + print() + params = {'max_results': 5, 'tweet.fields': 'created_at'} + tweets = api.request(f'users/:{USER_ID}/bookmarks', params) + for t in tweets: + pprint(t) + + # Get next 5 tweets + print() + next_token = tweets.json()['meta']['next_token'] + params = {'max_results': 5, 'tweet.fields': 'created_at', 'pagination_token': next_token} + tweets = api.request(f'users/:{USER_ID}/bookmarks', params) + for t in tweets: + pprint(t) + + +except TwitterRequestError as e: + print('Request error') + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print('Connection error') + print(e) + +except Exception as e: + print('Exception') + print(e) \ No newline at end of file From 3b4d86aae658bc86ac33f485bca0bd6885d081ab Mon Sep 17 00:00:00 2001 From: Sebastian Steins Date: Thu, 1 Sep 2022 12:59:37 +0200 Subject: [PATCH 134/139] Add support for known OAuth2 token to enable user-context requests Closes #219 Closes #220 --- TwitterAPI/BearerAuthUser.py | 40 ++++------------------------------- TwitterAPI/TwitterAPI.py | 4 +--- examples/v2/user_bookmarks.py | 2 +- 3 files changed, 6 insertions(+), 40 deletions(-) diff --git a/TwitterAPI/BearerAuthUser.py b/TwitterAPI/BearerAuthUser.py index 1482cf2..8f932c8 100644 --- a/TwitterAPI/BearerAuthUser.py +++ b/TwitterAPI/BearerAuthUser.py @@ -4,14 +4,9 @@ from .constants import * -import base64 import requests -OAUTH2_SUBDOMAIN = 'api' -OAUTH2_ENDPOINT = 'oauth2/token' - - class BearerAuthUser(requests.auth.AuthBase): """Request bearer access token for oAuth2 authentication. @@ -21,44 +16,17 @@ class BearerAuthUser(requests.auth.AuthBase): :param proxies: Dictionary of proxy URLs (see documentation for python-requests). """ - def __init__(self, consumer_key, consumer_secret, oauth2_access_token, proxies=None, user_agent=None): - self._consumer_key = consumer_key - self._consumer_secret = consumer_secret - self._access_token = oauth2_access_token + def __init__(self, oauth2_access_token, proxies=None, user_agent=None): self.proxies = proxies self.user_agent = user_agent - self._bearer_token = self._get_access_token() + self._bearer_token = oauth2_access_token - def _get_access_token(self): - token_url = '%s://%s.%s/%s' % (PROTOCOL, - OAUTH2_SUBDOMAIN, - DOMAIN, - OAUTH2_ENDPOINT) - auth = self._consumer_key + ':' + self._consumer_secret - b64_bearer_token_creds = base64.b64encode(auth.encode('utf8')) - params = {'grant_type': 'client_credentials'} - headers = {} - headers['User-Agent'] = self.user_agent - headers['Authorization'] = 'Basic ' + b64_bearer_token_creds.decode('utf8') - headers['Content-Type'] = 'application/x-www-form-urlencoded;charset=UTF-8' - try: - response = requests.post( - token_url, - params=params, - headers=headers, - proxies=self.proxies) - data = response.json() - return data['access_token'] - except Exception as e: - raise Exception('Error requesting bearer access token: %s' % e) def __call__(self, r): auth_list = [ - self._consumer_key, - self._consumer_secret, - self._access_token] + self._bearer_token] if all(auth_list): - r.headers['Authorization'] = "Bearer %s" % self._access_token + r.headers['Authorization'] = "Bearer %s" % self._bearer_token return r else: raise Exception('Not enough keys passed to Bearer token manager.') diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 5656fce..f14d879 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -98,11 +98,9 @@ def __init__( proxies=self.proxies, user_agent=self.USER_AGENT) elif auth_type == OAuthType.OAUTH2USER or auth_type == 'oAuth2User': - if not all([consumer_key, consumer_secret, oauth2_access_token]): + if not all([oauth2_access_token]): raise Exception('Missing authentication parameter') self.auth = OAuth2User( - consumer_key, - consumer_secret, oauth2_access_token=oauth2_access_token, proxies=self.proxies, user_agent=self.USER_AGENT) diff --git a/examples/v2/user_bookmarks.py b/examples/v2/user_bookmarks.py index a8ffc09..87cbbe1 100644 --- a/examples/v2/user_bookmarks.py +++ b/examples/v2/user_bookmarks.py @@ -6,7 +6,7 @@ try: o = TwitterOAuth.read_file() - api = TwitterAPI(o.consumer_key, o.consumer_secret, oauth2_access_token=access_token, api_version='2', auth_type="oAuth2User") + api = TwitterAPI(oauth2_access_token=access_token, api_version='2', auth_type="oAuth2User") # Get tweets - default setting tweets = api.request(f'users/:{USER_ID}/bookmarks') From 866752b9d1b8cbfee563d4a89c95a7bcfad862e5 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sun, 16 Oct 2022 22:42:22 -0400 Subject: [PATCH 135/139] Merged codewithbas-feature/user-bookmarks --- CHANGE.log | 4 +++- TwitterAPI/__init__.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index 2e32139..626d0a4 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -199,4 +199,6 @@ v2.7.11, 02 Jan 2022 -- Merged m0namon-counts_all_next_token v2.7.12, 11 Feb 2022 -- Merged Dheavyman-update-ads-api-v10 -v2.7.13, 06 Jun 2022 -- Fixed endpoint conflict in constanst.py \ No newline at end of file +v2.7.13, 06 Jun 2022 -- Fixed endpoint conflict in constanst.py + +v2.8.0, 16 Oct 2022 -- Merged codewithbas-feature/user-bookmarks \ No newline at end of file diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index e5ab050..d5c2520 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.7.13' +__version__ = '2.8.0' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 079d28e8e06f5409c940bd6fcd8a784b0407a2b0 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Sun, 16 Oct 2022 22:47:57 -0400 Subject: [PATCH 136/139] Merged Dheavyman-ads-api-v11 --- CHANGE.log | 4 +++- TwitterAPI/__init__.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index 626d0a4..4520b9a 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -201,4 +201,6 @@ v2.7.12, 11 Feb 2022 -- Merged Dheavyman-update-ads-api-v10 v2.7.13, 06 Jun 2022 -- Fixed endpoint conflict in constanst.py -v2.8.0, 16 Oct 2022 -- Merged codewithbas-feature/user-bookmarks \ No newline at end of file +v2.8.0, 16 Oct 2022 -- Merged codewithbas-feature/user-bookmarks + +v2.8.1, 16 Oct 2022 -- Merged Dheavyman-ads-api-v11 \ No newline at end of file diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index d5c2520..ca1509b 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.8.0' +__version__ = '2.8.1' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' From 5c07c02fce51302e747190a47b6c6796fee23352 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Mon, 5 Dec 2022 19:35:29 -0500 Subject: [PATCH 137/139] Added new example v2/stream_forever.py --- TwitterAPI/__init__.py | 2 +- examples/v2/stream_forever.py | 120 ++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 examples/v2/stream_forever.py diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index ca1509b..1126504 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.8.1' +__version__ = '2.8.1.1' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' diff --git a/examples/v2/stream_forever.py b/examples/v2/stream_forever.py new file mode 100644 index 0000000..67e19d7 --- /dev/null +++ b/examples/v2/stream_forever.py @@ -0,0 +1,120 @@ +from TwitterAPI import (TwitterAPI, + TwitterOAuth, + TwitterRequestError, + TwitterConnectionError, + HydrateType) + + +RULES = ['pizza','lambrusco'] + +EXPANSIONS = 'author_id' +USER_FIELDS = 'created_at,description,location,name,username' +TWEET_FIELDS = 'author_id,created_at,entities,id,lang,public_metrics,source,text' + + +## +## SET UP LOGGING TO FILE AND TO CONSOLE +## + +import logging +formatter = logging.Formatter('%(levelname)s %(asctime)s %(message)s', '%m/%d/%Y %I:%M:%S %p') +fh = logging.FileHandler('stream_forever.log') +sh = logging.StreamHandler() +fh.setFormatter(formatter) +sh.setFormatter(formatter) +logger = logging.getLogger() +logger.setLevel(logging.INFO) +logger.addHandler(fh) +logger.addHandler(sh) + + +## +## AUTHENTICATE WITH TWITTER +## + +o = TwitterOAuth.read_file() +api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + + +## +## STREAMING RULES +## + +# ADD RULES (NO HARM RE-ADDING, BUT ONCE IS ENOUGH) +for rule in RULES: + r = api.request('tweets/search/stream/rules', {'add': [{'value':rule}]}) + print(f'[{r.status_code}] RULE ADDED: {r.text}') + if r.status_code != 201: exit() + +# PRINT RULES +r = api.request('tweets/search/stream/rules', method_override='GET') +print(f'\n[{r.status_code}] RULES:') +if r.status_code != 200: exit() +for item in r: + print(item['value']) + + +## +## "BACK OFF" STRATEGY FOR DISCONNECTS AND RATE LIMITS +## USE 60 SECOND INTERVAL FOR HTTP 429 ERRORS +## USE 5 SECOND INTERVAL FOR ALL OTHER HTTP ERRORS +## REFER TO: https://developer.twitter.com/en/docs/twitter-api/tweets/filtered-stream/integrate/handling-disconnections +## + +from time import sleep + +backOffCount = 0 + +def backOff(interval): + global backOffCount, logger + seconds = pow(2, backOffCount) * interval + backOffCount = min(backOffCount + 1, 6) + logger.info(f'Back off {seconds} seconds') + sleep(seconds) + + +## +## STREAM FOREVER... +## + +while True: + try: + # START STREAM + logger.info('START STREAM...') + r = api.request('tweets/search/stream', + { + 'expansions': EXPANSIONS, + 'user.fields': USER_FIELDS, + 'tweet.fields': TWEET_FIELDS + }, + hydrate_type=HydrateType.APPEND) + + if r.status_code == 200: + logger.info(f'[{r.status_code}] STREAM CONNECTED') + backOffCount = 0 + else: + logger.info(f'[{r.status_code}] FAILED TO CONNECT. REASON:\n{r.text}') + backOff(60 if r.status_code == 429 else 5) + continue + + for item in r: + if 'data' in item: + print(item['data']['text']) + elif 'errors' in item: + logger.error(item['errors']) + else: + # PROBABLY SHOULD NEVER BRANCH TO HERE + logger.warning(f'UNKNOWN ITEM TYPE: {item}') + + except TwitterConnectionError: + backOff(5) + continue + + except TwitterRequestError: + break + + except KeyboardInterrupt: + break + + finally: + logger.info('STREAM STOPPED') \ No newline at end of file From 2ede005f51e542f4214825cec3d2c146e2200456 Mon Sep 17 00:00:00 2001 From: Eduardo Rosendo Date: Wed, 22 Mar 2023 17:40:29 -0400 Subject: [PATCH 138/139] fix(TwitterAPI): Use the json attribute to attach metadata This commit tweaks the request method to use the json attribute instead of the data attribute to attach additional metadata using the media/metadata/create endpoint from the v1.1 API. --- TwitterAPI/TwitterAPI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index f14d879..048f46c 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -201,7 +201,7 @@ def request(self, resource, params=None, files=None, method_override=None, hydra timeout = self.REST_TIMEOUT d = p = j = None if method == 'POST': - if self.version == '1.1': + if self.version == '1.1' and 'metadata' not in resource: d = params else: j = params From ffa9cad2903a0b6c29c7d8da2f7718ecdf70d111 Mon Sep 17 00:00:00 2001 From: Jonas Geduldig Date: Wed, 22 Mar 2023 22:02:36 -0400 Subject: [PATCH 139/139] ERosendo PR --- CHANGE.log | 4 +++- TwitterAPI/__init__.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGE.log b/CHANGE.log index 4520b9a..b9426a9 100644 --- a/CHANGE.log +++ b/CHANGE.log @@ -203,4 +203,6 @@ v2.7.13, 06 Jun 2022 -- Fixed endpoint conflict in constanst.py v2.8.0, 16 Oct 2022 -- Merged codewithbas-feature/user-bookmarks -v2.8.1, 16 Oct 2022 -- Merged Dheavyman-ads-api-v11 \ No newline at end of file +v2.8.1, 16 Oct 2022 -- Merged Dheavyman-ads-api-v11 + +v2.8.2, 22 Mar 2023 -- Merged ERosendo PR diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index 1126504..ae9e26c 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.8.1.1' +__version__ = '2.8.2' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig'