diff --git a/.gitignore b/.gitignore index 58e178d..719cb9d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ MANIFEST +.DS_Store +.vscode/* + *.py[cod] # C extensions @@ -35,3 +38,5 @@ nosetests.xml .mr.developer.cfg .project .pydevproject +.idea/* +env/* diff --git a/CHANGES.txt b/CHANGE.log similarity index 50% rename from CHANGES.txt rename to CHANGE.log index 48de199..b9426a9 100644 --- a/CHANGES.txt +++ b/CHANGE.log @@ -106,3 +106,103 @@ v2.4.6, 16 Jul 2017 -- Merged RickRedSix patch. Added direct message endpoint v2.4.7, 22 Oct 2017 -- Added endpoints for Ads API. v2.4.8, 03 Feb 2018 -- Merged RickRedSix patch. Added account activity endpoints. + +v2.4.9, 03 Feb 2018 -- Merged torufurukawa patch. Added webhook endpoint. + +v2.4.10, 11 Mar 2018 -- Merged torufurukawa patch. Added direct message endpoint. + +v2.5.0, 15 Mar 2018 -- Renamed get_rest_quota() to get_quota(). + Renamed TwitterRestPager to TwitterPager. + +v2.5.1, 19 May 2018 -- Added support for both HTTP and HTTPS proxies. + +v2.5.2, 19 May 2018 -- Added endpoints for Premium Search API. + +v2.5.3, 19 May 2018 -- Added 'results' to iterator and a new Premium Search example. + +v2.5.4, 25 May 2018 -- Modified TwitterPager to support Premium Search's cursor. + +v2.5.5, 07 Oct 2018 -- Merged x0139 patch. Added typing indicator endpoint. + Added DM destroy endpoint. + +v2.5.6, 09 Oct 2018 -- Fixed TwitterPager. + +v2.5.7, 15 Dec 2018 -- Merged HyperManTT patch. Added subscripts/list endpoint. + +v2.5.8, 25 Dec 2018 -- Fixed bug in TwitterPager, repeating results. + +v2.5.9, 14 Jan 2019 -- Fixed bug in TwitterPager, stalling but not exiting when no more results. + Moved USER_AGENT and TIMEOUT constants to TwitterAPI class static variables. + +v2.5.10, 28 Oct 2019 -- Merged HammadH-variable-timeouts. + +v2.5.11, 10 Apr 2020 -- Merged RickRedSix patch. Added Labs API. + +v2.5.12, 13 Jul 2020 -- Merged drbig patch. Improved error reporting. + +v2.5.13, 17 Jul 2020 -- Merged drbig patch. Override exception handling in TwitterPager. + Merged reuning patch. Added new endpoint. + +v2.5.14, 17 Sep 2020 -- Fix TwitterPager to iterate integer ids. + +v2.6.0, 04 Oct 2020 -- Merged dylancaponi-twitter-v2 with V2 support. + Made several modifications to complete V2 support. + Dropping Python 2.7 compatibility. + +v2.6.1, 23 Oct 2020 -- Fix TwitterPager, add conversation example. + +v2.6.2, 27 Oct 2020 -- Merged ckovamees-master with add media endpoints. + +v2.6.3, 12 Jan 2021 -- Merged peguerosdc-master with v2 user endpoints. + +v2.6.4, 27 Jan 2021 -- Fixed conversation_tree.py example. + +v2.6.5, 30 Jan 2021 -- Merged dylancaponi-patch-1 with Ads endpoints. + +v2.6.6, 14 Feb 2021 -- Fixed v2 next/prev token in TwitterPager. + +v2.6.7, 19 Feb 2021 -- Fixed v2 next/prev token in TwitterPager for search. + +v2.6.8, 19 Feb 2021 -- Added v2 tweet hydration. + +v2.6.9, 21 Feb 2021 -- Fixed bug in _StreamingIterable. + +v2.6.10, 06 Apr 2021 -- Added 'mentions' to be hydrated. + +v2.7.0, 12 Apr 2021 -- Two types of hydrate: APPEND and REPLACE. + Two new enums: HydrateType, OAuthType. + +v2.7.1, 14 Apr 2021 -- Fixed oAuth version resolution. + +v2.7.2, 22 Apr 2021 -- Fixed _StreamIterable for v2. + +v2.7.3, 26 Apr 2021 -- Fixed hydrate for media_keys. + +v2.7.4, 27 Jun 2021 -- Merged RojerGS-patch-1 to fix documentation links. + Merged peguerosdc-endpoint/v2-liking to add support for V2 likes. + Changed 'stall_warning' to 'stall_warnings'. + +v2.7.5, 18 Jul 2021 -- Merged Dheavyman-fix-meta-keyerror. + +v2.7.6, 11 Nov 2021 -- Merged dylancaponi-patch-2 to update ADS version. + +v2.7.7, 11 Nov 2021 -- Merged NeodymiumFerBore-patch-1, musicist288-master + +v2.7.8, 29 Nov 2021 -- Fixed hydrated media keys + +v2.7.9, 30 Nov 2021 -- Specify all supported endpoint methods in constants.py + Added new v2 endpoints + +v2.7.10, 14 Dec 2021 -- Fixed v1.1 POST bug + +v2.7.11, 02 Jan 2022 -- Merged m0namon-counts_all_next_token + +v2.7.12, 11 Feb 2022 -- Merged Dheavyman-update-ads-api-v10 + +v2.7.13, 06 Jun 2022 -- Fixed endpoint conflict in constanst.py + +v2.8.0, 16 Oct 2022 -- Merged codewithbas-feature/user-bookmarks + +v2.8.1, 16 Oct 2022 -- Merged Dheavyman-ads-api-v11 + +v2.8.2, 22 Mar 2023 -- Merged ERosendo PR diff --git a/README.rst b/README.rst index 211d9a0..34a45d5 100755 --- a/README.rst +++ b/README.rst @@ -1,7 +1,5 @@ |LOGO| -============================ -|BADGE_VERSION| |BADGE_CHAT| -============================ +====== .. |LOGO| image:: https://raw.githubusercontent.com/geduldig/TwitterAPI/master/logo.png .. |BADGE_VERSION| image:: http://img.shields.io/pypi/v/TwitterAPI.svg @@ -10,57 +8,83 @@ :alt: Join the chat at https://gitter.im/geduldig/TwitterAPI :target: https://gitter.im/geduldig/TwitterAPI?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge -TwitterAPI is a Python package for accessing Twitter's REST APIs and Streaming APIs. It supports OAuth 1.0 and OAuth 2.0 authentication. And, it works with the latest Python versions in both 2.x and 3.x branches. +.. |BADGE_2| image:: https://img.shields.io/endpoint?url=https%3A%2F%2Fblue-sea-697d.quartiers047.workers.dev%3A443%2Fhttps%2Ftwbadges.glitch.me%2Fbadges%2Fv2 + :target: https://developer.twitter.com/en/docs/twitter-api +.. |BADGE_LABS| image:: https://img.shields.io/endpoint?url=https%3A%2F%2Fblue-sea-697d.quartiers047.workers.dev%3A443%2Fhttps%2Ftwbadges.glitch.me%2Fbadges%2Flabs + :target: https://developer.twitter.com/en/docs/labs +.. |BADGE_ADS| image:: https://img.shields.io/endpoint?url=https%3A%2F%2Fblue-sea-697d.quartiers047.workers.dev%3A443%2Fhttps%2Ftwbadges.glitch.me%2Fbadges%2Fadsv9 + :target: https://developer.twitter.com/en/docs/twitter-ads-api +.. |BADGE_1.1| image:: https://img.shields.io/endpoint?url=https%3A%2F%2Fblue-sea-697d.quartiers047.workers.dev%3A443%2Fhttps%2Ftwbadges.glitch.me%2Fbadges%2Fstandard + :target: https://developer.twitter.com/en/docs/twitter-api +.. |BADGE_PREMIUM| image:: https://img.shields.io/endpoint?url=https%3A%2F%2Fblue-sea-697d.quartiers047.workers.dev%3A443%2Fhttps%2Ftwbadges.glitch.me%2Fbadges%2Fpremium + :target: https://developer.twitter.com + + +TwitterAPI is a minimal python wrapper for the TwitterAPIs. A list of what it can do: + +* Support for all V1.1 and V2 endpoints, plus Premium, Ads, Labs, Collections. +* OAuth1 and bearer token authentication, and proxy server authentication. +* Streaming endpoints. +* Paging results. +* The option to "hydrate" results returned by V2 endpoints. +* Error handling. Installation ------------ -From the command line:: - pip install TwitterAPI + > pip install TwitterAPI -Documentation -------------- -* `An Introduction `_ -* `Authentication `_ -* `Error Handling `_ -* `Paging Results `_ -* `Tiny Examples `_ -* `Fault Tolerant Streams and Pages `_ - -Some Code... ------------- -[See `TwitterAPI/examples `_ for working examples.] +Twitter API Version 1.1 Code Snippets +------------------------------------- +[More examples in `TwitterAPI/examples/v1.1 `_] -First, authenticate with your application credentials:: +Search for recent tweets +:: from TwitterAPI import TwitterAPI api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret) + r = api.request('search/tweets', {'q':'pizza'}) + for item in r: + print(item) -Tweet something:: - - r = api.request('statuses/update', {'status':'This is a tweet!'}) - print(r.status_code) +Stream tweets from New York City as they get tweeted +:: -Get tweet by its id:: + r = api.request('statuses/filter', {'locations':'-74,40,-73,41'}) + for item in r: + print(item) - r = api.request('statuses/show/:%d' % 210462857140252672) - print(r.text) +Twitter API Version 2 Code Snippets +------------------------------------ +[More examples in `TwitterAPI/examples/v2 `_ ] -Get some tweets:: +Search for recent tweets, and specify `fields` and `expansions` +:: - r = api.request('search/tweets', {'q':'pizza'}) + from TwitterAPI import TwitterAPI + api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret, api_version='2') + r = api.request('tweets/search/recent', { + 'query':'pizza', + 'tweet.fields':'author_id', + 'expansions':'author_id'}) for item in r: print(item) -Stream tweets from New York City:: +One Method For Everything +------------------------- - r = api.request('statuses/filter', {'locations':'-74,40,-73,41'}) - for item in r: - print(item) - -Notice that ``request()`` works with all endpoints found in either the REST APIs or the Streaming APIs. Usually ``request()`` takes two arguments: a Twitter endpoint and a dictionary of endpoint parameters. The above examples use ``get_iterator()`` to consume each tweet object. The iterator knows how to iterate results returned from either the REST APIs or the Streaming APIs. +The ``request()`` method works with all version 1.1 and version 2 endpoints. Typcally, ``request()`` takes two arguments: a Twitter endpoint and a dictionary of endpoint parameters. + +The method returns an object that will iterate either search results and streams. The returned object also gives you access to the raw response (``r.text``) and the HTTP status code (``r.status_code``). See the `requests `_ library documentation for more details. -You also have access to the response object returned by ``request()``. From a response object ``r`` you can get the raw response with ``r.text`` and the HTTP status code with ``r.status_code``. See the `requests `_ library documentation for more details. +Documentation +------------- +* `An Introduction `_ +* `Authentication `_ +* `Error Handling `_ +* `Paging Results `_ +* `Tiny Examples `_ +* `Fault Tolerant Streams and Pages `_ Extra Goodies ------------- diff --git a/TODO.txt b/TODO.txt index 5ee980e..dc023b8 100644 --- a/TODO.txt +++ b/TODO.txt @@ -2,3 +2,5 @@ Build test suite with unittest. (If anyone wants to contribute, would be much appreciated.) Test Ads API endpoints. + +Add and test Enterprise Search API endpoints. diff --git a/TwitterAPI/BearerAuth.py b/TwitterAPI/BearerAuth.py index 335c805..be487bc 100644 --- a/TwitterAPI/BearerAuth.py +++ b/TwitterAPI/BearerAuth.py @@ -21,10 +21,11 @@ class BearerAuth(requests.auth.AuthBase): :param proxies: Dictionary of proxy URLs (see documentation for python-requests). """ - def __init__(self, consumer_key, consumer_secret, proxies=None): + def __init__(self, consumer_key, consumer_secret, proxies=None, user_agent=None): self._consumer_key = consumer_key self._consumer_secret = consumer_secret self.proxies = proxies + self.user_agent = user_agent self._bearer_token = self._get_access_token() def _get_access_token(self): @@ -36,7 +37,7 @@ def _get_access_token(self): b64_bearer_token_creds = base64.b64encode(auth.encode('utf8')) params = {'grant_type': 'client_credentials'} headers = {} - headers['User-Agent'] = USER_AGENT + headers['User-Agent'] = self.user_agent headers['Authorization'] = 'Basic ' + b64_bearer_token_creds.decode('utf8') headers['Content-Type'] = 'application/x-www-form-urlencoded;charset=UTF-8' try: diff --git a/TwitterAPI/BearerAuthUser.py b/TwitterAPI/BearerAuthUser.py new file mode 100644 index 0000000..8f932c8 --- /dev/null +++ b/TwitterAPI/BearerAuthUser.py @@ -0,0 +1,32 @@ +__author__ = "Andrea Biancini, geduldig" +__date__ = "January 3, 2014" +__license__ = "MIT" + + +from .constants import * +import requests + + +class BearerAuthUser(requests.auth.AuthBase): + + """Request bearer access token for oAuth2 authentication. + + :param consumer_key: Twitter application consumer key + :param consumer_secret: Twitter application consumer secret + :param proxies: Dictionary of proxy URLs (see documentation for python-requests). + """ + + def __init__(self, oauth2_access_token, proxies=None, user_agent=None): + self.proxies = proxies + self.user_agent = user_agent + self._bearer_token = oauth2_access_token + + + def __call__(self, r): + auth_list = [ + self._bearer_token] + if all(auth_list): + r.headers['Authorization'] = "Bearer %s" % self._bearer_token + return r + else: + raise Exception('Not enough keys passed to Bearer token manager.') diff --git a/TwitterAPI/TwitterAPI.py b/TwitterAPI/TwitterAPI.py index 407e692..048f46c 100644 --- a/TwitterAPI/TwitterAPI.py +++ b/TwitterAPI/TwitterAPI.py @@ -4,19 +4,40 @@ from .BearerAuth import BearerAuth as OAuth2 +from .BearerAuthUser import BearerAuthUser as OAuth2User from .constants import * +from .TwitterError import * from datetime import datetime +from enum import Enum from requests.exceptions import ConnectionError, ReadTimeout, SSLError from requests.packages.urllib3.exceptions import ReadTimeoutError, ProtocolError from requests_oauthlib import OAuth1 -from .TwitterError import * import json +import os import requests import socket import ssl import time +DEFAULT_USER_AGENT = os.getenv('DEFAULT_USER_AGENT', 'python-TwitterAPI') +DEFAULT_CONNECTION_TIMEOUT = os.getenv('DEFAULT_CONNECTION_TIMEOUT', 5) +DEFAULT_STREAMING_TIMEOUT = os.getenv('DEFAULT_STREAMING_TIMEOUT', 90) +DEFAULT_REST_TIMEOUT = os.getenv('DEFAULT_REST_TIMEOUT', 5) + + +class OAuthType(Enum): + OAUTH1 = 'oAuth1' + OAUTH2 = 'oAuth2' + OAUTH2USER = 'oAuth2User' + + +class HydrateType(Enum): + NONE = 0 + APPEND = 1 + REPLACE = 2 + + class TwitterAPI(object): """Access REST API or Streaming API resources. @@ -26,20 +47,41 @@ class TwitterAPI(object): :param access_token_key: Twitter application access token key :param access_token_secret: Twitter application access token secret :param auth_type: "oAuth1" (default) or "oAuth2" - :param proxy_url: HTTPS proxy URL (ex. "https://USER:PASSWORD@SERVER:PORT") + :param proxy_url: HTTPS proxy URL string (ex. 'https://USER:PASSWORD@SERVER:PORT'), + or dict of URLs (ex. {'http':'http://SERVER', 'https':'https://SERVER'}) """ + # static properties to be overridden if desired + USER_AGENT = DEFAULT_USER_AGENT + CONNECTION_TIMEOUT = DEFAULT_CONNECTION_TIMEOUT + STREAMING_TIMEOUT = DEFAULT_STREAMING_TIMEOUT + REST_TIMEOUT = DEFAULT_REST_TIMEOUT + def __init__( self, consumer_key=None, consumer_secret=None, access_token_key=None, access_token_secret=None, - auth_type='oAuth1', - proxy_url=None): + oauth2_access_token=None, + auth_type=OAuthType.OAUTH1, + proxy_url=None, + api_version=VERSION): """Initialize with your Twitter application credentials""" - self.proxies = {'https': proxy_url} if proxy_url else None - if auth_type == 'oAuth1': + # if there are multiple API versions, this will be the default version which can + # also be overridden by specifying the version when calling the request method. + self.version = api_version + + # Optional proxy or proxies. + if isinstance(proxy_url, dict): + self.proxies = proxy_url + elif proxy_url is not None: + self.proxies = {'https': proxy_url} + else: + self.proxies = None + + # Twitter supports two types of authentication. + if auth_type == OAuthType.OAUTH1 or auth_type == 'oAuth1': if not all([consumer_key, consumer_secret, access_token_key, access_token_secret]): raise Exception('Missing authentication parameter') self.auth = OAuth1( @@ -47,13 +89,21 @@ def __init__( consumer_secret, access_token_key, access_token_secret) - elif auth_type == 'oAuth2': + elif auth_type == OAuthType.OAUTH2 or auth_type == 'oAuth2': if not all([consumer_key, consumer_secret]): raise Exception('Missing authentication parameter') self.auth = OAuth2( consumer_key, consumer_secret, - proxies=self.proxies) + proxies=self.proxies, + user_agent=self.USER_AGENT) + elif auth_type == OAuthType.OAUTH2USER or auth_type == 'oAuth2User': + if not all([oauth2_access_token]): + raise Exception('Missing authentication parameter') + self.auth = OAuth2User( + oauth2_access_token=oauth2_access_token, + proxies=self.proxies, + user_agent=self.USER_AGENT) else: raise Exception('Unknown oAuth version') @@ -70,12 +120,25 @@ def _prepare_url(self, subdomain, path): DOMAIN, ADS_VERSION, path) - else: + elif subdomain == 'api' and 'labs/' in path: + return '%s://%s.%s/%s' % (PROTOCOL, + subdomain, + DOMAIN, + path) + elif self.version == '1.1': return '%s://%s.%s/%s/%s.json' % (PROTOCOL, subdomain, DOMAIN, - VERSION, + self.version, path) + elif self.version == '2': + return '%s://%s.%s/%s/%s' % (PROTOCOL, + subdomain, + DOMAIN, + self.version, + path) + else: + raise Exception('Unsupported API version') def _get_endpoint(self, resource): """Substitute any parameters in the resource path with :PARAM.""" @@ -89,56 +152,92 @@ def _get_endpoint(self, resource): else: return (resource, resource) - def request(self, resource, params=None, files=None, method_override=None): + def request(self, resource, params=None, files=None, method_override=None, hydrate_type=HydrateType.NONE): """Request a Twitter REST API or Streaming API resource. - :param resource: A valid Twitter endpoint (ex. "search/tweets") + :param resource: A Twitter endpoint (ex. "search/tweets") :param params: Dictionary with endpoint parameters or None (default) :param files: Dictionary with multipart-encoded file or None (default) - :param method_override: Request method to override or None (default) + :param method_override: Request method to override or None (default). + If an endpoint supports more than one method, the default method is the first in the list of methods. + :param hydrate_type: HydrateType or int + Do not hydrate ('includes' field and all its content will be lost for non-streaming requests) - NONE or 0 (default) + Append new field with '_hydrate' suffix with hydrate values - APPEND or 1 + Replace current field value with hydrate values - REPLACE or 2 :returns: TwitterResponse :raises: TwitterConnectionError """ + # check that the endpoint is valid resource, endpoint = self._get_endpoint(resource) if endpoint not in ENDPOINTS: raise Exception('Endpoint "%s" unsupported' % endpoint) + # check that the method is valid if the endpoint supports more than one method + method, subdomain = ENDPOINTS[endpoint] + if not method_override and isinstance(method, list): + method = method[0] # by default use first method in list + elif isinstance(method, list): + if method_override in method: + method = method_override # use method_override + else: + raise Exception(f'Endpoint "{endpoint}" with method "{method_override}" unsupported') + with requests.Session() as session: session.auth = self.auth - session.headers = {'User-Agent': USER_AGENT} - method, subdomain = ENDPOINTS[endpoint] - if method_override: - method = method_override + session.headers = {'User-Agent': self.USER_AGENT} url = self._prepare_url(subdomain, resource) - if 'stream' in subdomain: + if self.version == '1.1' and 'stream' in subdomain: session.stream = True - timeout = STREAMING_TIMEOUT - # always use 'delimited' for efficient stream parsing + timeout = self.STREAMING_TIMEOUT if not params: params = {} params['delimited'] = 'length' - params['stall_warning'] = 'true' + params['stall_warnings'] = 'true' + elif self.version == '2' and resource.endswith('/stream'): + session.stream = True + timeout = self.STREAMING_TIMEOUT else: session.stream = False - timeout = REST_TIMEOUT + timeout = self.REST_TIMEOUT + d = p = j = None if method == 'POST': - data = params - params = None + if self.version == '1.1' and 'metadata' not in resource: + d = params + else: + j = params + elif method == 'PUT': + j = params else: - data = None - try: - r = session.request( - method, - url, - data=data, - params=params, - timeout=(CONNECTION_TIMEOUT,timeout), - files=files, - proxies=self.proxies) + p = params + + try: + if False and method == 'PUT': + session.headers['Content-type'] = 'application/json' + data = params + r = session.request( + method, + url, + json=data) + else: + r = session.request( + method, + url, + data=d, + params=p, + json=j, + timeout=(self.CONNECTION_TIMEOUT, timeout), + files=files, + proxies=self.proxies) except (ConnectionError, ProtocolError, ReadTimeout, ReadTimeoutError, SSLError, ssl.SSLError, socket.error) as e: raise TwitterConnectionError(e) - return TwitterResponse(r, session.stream) + + options = { + 'api_version': self.version, + 'is_stream': session.stream, + 'hydrate_type': hydrate_type + } + return TwitterResponse(r, options) class TwitterResponse(object): @@ -147,11 +246,12 @@ class TwitterResponse(object): :param response: The requests.Response object returned by the API call :param stream: Boolean connection type (True if a streaming connection) + :param options: Dict containing parsing options """ - def __init__(self, response, stream): + def __init__(self, response, options): self.response = response - self.stream = stream + self.options = options @property def headers(self): @@ -184,12 +284,12 @@ def get_iterator(self): :raises: TwitterConnectionError, TwitterRequestError """ if self.response.status_code != 200: - raise TwitterRequestError(self.response.status_code) + raise TwitterRequestError(self.response.status_code, msg=self.response.text) - if self.stream: - return iter(_StreamingIterable(self.response)) + if self.options['is_stream']: + return iter(_StreamingIterable(self.response, self.options)) else: - return iter(_RestIterable(self.response)) + return iter(_RestIterable(self.response, self.options)) def __iter__(self): """Get API dependent iterator. @@ -199,7 +299,7 @@ def __iter__(self): """ return self.get_iterator() - def get_rest_quota(self): + def get_quota(self): """Quota information in the REST-only response header. :returns: Dictionary of 'remaining' (count), 'limit' (count), 'reset' (time) @@ -225,31 +325,54 @@ class _RestIterable(object): """Iterate statuses, errors or other iterable objects in a REST API response. :param response: The request.Response from a Twitter REST API request + :param options: Dict containing parsing options """ - def __init__(self, response): + def __init__(self, response, options): resp = response.json() - # convert json response into something iterable - if 'errors' in resp: - self.results = resp['errors'] - elif 'statuses' in resp: - self.results = resp['statuses'] - elif 'users' in resp: - self.results = resp['users'] - elif 'ids' in resp: - self.results = resp['ids'] - elif 'data' in resp and not isinstance(resp['data'], dict): - self.results = resp['data'] - elif hasattr(resp, '__iter__') and not isinstance(resp, dict): - if len(resp) > 0 and 'trends' in resp[0]: - self.results = resp[0]['trends'] + if options['api_version'] == '2': + if 'data' in resp: + if isinstance(resp['data'], dict): + resp['data'] = [resp['data']] + h_type = options['hydrate_type'] + if 'includes' in resp and h_type != HydrateType.NONE: + field_suffix = '' if h_type == HydrateType.REPLACE else '_hydrate' + self.results = _hydrate_tweets(resp['data'], resp['includes'], field_suffix) + else: + self.results = resp['data'] else: - self.results = resp + self.results = [] + elif options['api_version'] == '1.1': + # convert json response into something iterable + if 'errors' in resp: + self.results = resp['errors'] + elif 'statuses' in resp: + self.results = resp['statuses'] + elif 'users' in resp: + self.results = resp['users'] + elif 'ids' in resp: + self.results = resp['ids'] + elif 'results' in resp: + self.results = resp['results'] + elif 'data' in resp: + if not isinstance(resp['data'], dict): + self.results = resp['data'] + else: + self.results = [resp['data']] + elif hasattr(resp, '__iter__') and not isinstance(resp, dict): + if len(resp) > 0 and 'trends' in resp[0]: + self.results = resp[0]['trends'] + else: + self.results = resp + else: + self.results = (resp,) else: - self.results = (resp,) + self.results = [] def __iter__(self): - """Return a tweet status as a JSON object.""" + """ + :returns: Tweet status as a JSON object. + """ for item in self.results: yield item @@ -259,10 +382,12 @@ class _StreamingIterable(object): """Iterate statuses or other objects in a Streaming API response. :param response: The request.Response from a Twitter Streaming API request + :param options: Dict containing parsing options """ - def __init__(self, response): + def __init__(self, response, options): self.stream = response.raw + self.options = options def _iter_stream(self): """Stream parser. @@ -282,7 +407,7 @@ def _iter_stream(self): # check for stall (i.e. no data for 90 seconds) if not stall_timer: stall_timer = time.time() - elif time.time() - stall_timer > STREAMING_TIMEOUT: + elif time.time() - stall_timer > TwitterAPI.STREAMING_TIMEOUT: raise TwitterConnectionError('Twitter stream stalled') elif stall_timer: stall_timer = None @@ -294,24 +419,88 @@ def _iter_stream(self): item = None item = self.stream.read(nbytes) break + if item: + item = json.loads(item.decode('utf8')) + if self.options['api_version'] == '2': + h_type = self.options['hydrate_type'] + if h_type != HydrateType.NONE: + if 'data' in item and 'includes' in item: + field_suffix = '' if h_type == HydrateType.REPLACE else '_hydrate' + item = { 'data':_hydrate_tweets(item['data'], item['includes'], field_suffix) } yield item - except (ConnectionError, ProtocolError, ReadTimeout, ReadTimeoutError, - SSLError, ssl.SSLError, socket.error) as e: + except (ConnectionError, ValueError, ProtocolError, ReadTimeout, ReadTimeoutError, + SSLError, ssl.SSLError, socket.error, json.decoder.JSONDecodeError) as e: raise TwitterConnectionError(e) except AttributeError: # inform iterator to exit when client closes connection raise StopIteration def __iter__(self): - """Iterator. - + """ :returns: Tweet status as a JSON object. :raises: TwitterConnectionError """ for item in self._iter_stream(): if item: - try: - yield json.loads(item.decode('utf8')) - except ValueError as e: - # invalid JSON string (possibly an unformatted error message) - raise TwitterConnectionError(e) + yield item + + +def _hydrate_tweets(data, includes, field_suffix): + """Insert expansion fields back into tweet data by appending + a new field as a sibling to the referenced field. + + :param data: "data" property value in JSON response + :param includes: "includes" property value in JSON response + :param field_suffix: Suffix appended to a hydrated field name. + Either "_hydrate" which puts hydrated values into + a new field, or "" which replaces the current + field value with hydrated values. + + :returns: Tweet status as a JSON object. + """ + new_fields = [] + for key in includes: + incl = includes[key] + for obj in incl: + for field in ['id', 'media_key', 'username']: + if field in obj: + _create_include_fields(data, (obj[field], obj), new_fields) + + for item in new_fields: + parent = item[0] + field = item[1] + field_suffix + include = item[2] + if field in parent: + if item[1] == 'media_keys': + parent[field] += include + if field_suffix == '': + # REPLACE option + parent[field].remove(include[0]['media_key']) + else: + parent[field] = include + else: + parent[field] = include + return data + + +def _create_include_fields(parent, include, new_fields): + """Depth-first seach into 'parent' to locate fields referenced in + 'include'. Each match is appended to 'new_fields'. + """ + if isinstance(parent, list): + for item in parent: + _create_include_fields(item, include, new_fields) + elif isinstance(parent, dict): + for key, value in parent.items(): + if value == include[0]: + new_fields.append((parent, key, include[1])) + elif isinstance(value, list) and all(isinstance(elem, str) for elem in value): + # this code gets executed for "media_keys" which are in a list object + includes_list = [] + for elem in value: + if elem == include[0]: + includes_list.append(include[1]) + if len(includes_list) > 0: + new_fields.append((parent, key, includes_list)) + else: + _create_include_fields(value, include, new_fields) diff --git a/TwitterAPI/TwitterError.py b/TwitterAPI/TwitterError.py index 23010d8..0ebfa21 100644 --- a/TwitterAPI/TwitterError.py +++ b/TwitterAPI/TwitterError.py @@ -4,6 +4,7 @@ import logging +import json class TwitterError(Exception): @@ -17,7 +18,7 @@ class TwitterConnectionError(TwitterError): """Raised when the connection needs to be re-established""" def __init__(self, value): - super(Exception, self).__init__(value) + super().__init__(value) logging.warning('%s %s' % (type(value), value)) @@ -25,14 +26,30 @@ class TwitterRequestError(TwitterError): """Raised when request fails""" - def __init__(self, status_code): - if status_code >= 500: - msg = 'Twitter internal error (you may re-try)' - else: - msg = 'Twitter request failed' + def __init__(self, status_code, msg=None): + if msg is None: + if status_code >= 500: + msg = 'Twitter internal error (you may re-try)' + else: + msg = 'Twitter request failed' logging.info('Status code %d: %s' % (status_code, msg)) - super(Exception, self).__init__(msg) + super().__init__(msg) self.status_code = status_code - + self.msg = msg + def __str__(self): - return '%s (%d)' % (self.args[0], self.status_code) + return '%s (%d): %s' % (self.args, self.status_code, self.msg) + + + def __iter__(self): + try: + msg = json.loads(self.msg) + if 'errors' in msg: + for error in msg['errors']: + yield error['message'] + elif 'detail' in msg: + yield msg['detail'] + else: + yield self.msg + except: + yield self.msg \ No newline at end of file diff --git a/TwitterAPI/TwitterOAuth.py b/TwitterAPI/TwitterOAuth.py index 80953a6..1953827 100644 --- a/TwitterAPI/TwitterOAuth.py +++ b/TwitterAPI/TwitterOAuth.py @@ -50,10 +50,21 @@ def read_file(cls, file_name=None): oauth = {} for line in f: if '=' in line: - name, value = line.split('=', 1) - oauth[name.strip()] = value.strip() + tokens = line.strip().split('=', 1) + if len(tokens) != 2 or not tokens[1]: + raise Exception(cls.usage(file_name)) + oauth[tokens[0].strip()] = tokens[1].strip() return TwitterOAuth( oauth['consumer_key'], oauth['consumer_secret'], oauth['access_token_key'], oauth['access_token_secret']) + + @classmethod + def usage(cls, file_name): + return (f'Credentials file: {file_name}\n' + 'Expected format:\n' + 'consumer_key=YOUR_CONSUMER_KEY\n' + 'consumer_secret=YOUR_CONSUMER_SECRET\n' + 'access_token_key=YOUR_ACCESS_TOKEN\n' + 'access_token_secret=YOUR_ACCESS_TOKEN_SECRET\n') \ No newline at end of file diff --git a/TwitterAPI/TwitterPager.py b/TwitterAPI/TwitterPager.py new file mode 100644 index 0000000..816cde6 --- /dev/null +++ b/TwitterAPI/TwitterPager.py @@ -0,0 +1,138 @@ +__author__ = "geduldig" +__date__ = "June 8, 2013" +__license__ = "MIT" + + +from .TwitterAPI import HydrateType +from .TwitterError import * +from requests.exceptions import ConnectionError, ReadTimeout, SSLError +from requests.packages.urllib3.exceptions import ReadTimeoutError, ProtocolError +import requests +import time + + +class TwitterPager(object): + + """Continuous (stream-like) pagination of response from Twitter REST API resource. + In addition to Public API endpoints, supports Premium Search API. + + :param api: An authenticated TwitterAPI object + :param resource: String with the resource path (ex. search/tweets) + :param params: Dictionary of resource parameters + :param hydrate_type: HydrateType or int + Do not hydrate - NONE or 0 (default) + Append new field with '_hydrate' suffix with hydrate values - APPEND or 1 + Replace current field value with hydrate values - REPLACE or 2 + """ + + def __init__(self, api, resource, params=None, hydrate_type=HydrateType.NONE): + self.api = api + self.resource = resource + if not params: + params = {} + self.params = params + self.hydrate_type = hydrate_type + + def get_iterator(self, wait=5, new_tweets=False): + """Iterate response from Twitter REST API resource. Resource is called + in a loop to retrieve consecutive pages of results. + + :param wait: Floating point number (default=5) of seconds wait between requests. + Depending on the resource, appropriate values are 5 or 60 seconds. + :param new_tweets: Boolean determining the search direction. + False (default) retrieves old results. + True retrieves current results. + + :returns: JSON objects containing statuses, errors or other return info. + """ + elapsed = 0 + while True: + try: + # REQUEST ONE PAGE OF RESULTS... + start = time.time() + r = self.api.request(self.resource, self.params, hydrate_type=self.hydrate_type) + it = r.get_iterator() + if new_tweets: + it = reversed(list(it)) + + # YIELD FOR EACH ITEM IN THE PAGE... + item_count = 0 + id = None + for item in it: + item_count += 1 + if type(item) is dict: + if 'id' in item: + id = item['id'] + if 'code' in item: + if item['code'] in [130, 131]: + # Twitter service error + raise TwitterConnectionError(item) + yield item + + data = r.json() + + # CHECK FOR NEXT PAGE OR BAIL... + if self.api.version == '1.1': + # if a cursor is present, use it to get next page + # otherwise, use id to get next page + is_premium_search = self.params and 'query' in self.params and self.api.version == '1.1' + cursor = -1 + if new_tweets and 'previous_cursor' in data: + cursor = data['previous_cursor'] + cursor_param = 'cursor' + elif not new_tweets: + if 'next_cursor' in data: + cursor = data['next_cursor'] + cursor_param = 'cursor' + elif 'next' in data: + # 'next' is used by Premium Search (OLD searches only) + cursor = data['next'] + cursor_param = 'next' + + # bail when no more results + if cursor == 0: + break + elif cursor == -1 and is_premium_search: + break + elif not new_tweets and item_count == 0: + break + else: # VERSION 2 + if 'meta' not in data: + break + meta = data['meta'] + if not new_tweets and not 'next_token' in meta: + break + + # SLEEP... + elapsed = time.time() - start + pause = wait - elapsed if elapsed < wait else 0 + time.sleep(pause) + + # SETUP REQUEST FOR NEXT PAGE... + if self.api.version == '1.1': + # get a page with cursor if present, or with id if not + # a Premium search (i.e. 'query' is not a parameter) + if cursor != -1: + self.params[cursor_param] = cursor + elif id is not None and not is_premium_search: + if new_tweets: + self.params['since_id'] = str(id) + else: + self.params['max_id'] = str(id - 1) + else: + continue + else: # VERSION 2 + # TWITTER SHOULD STANDARDIZE ON pagination_token IN THE FUTURE + SEARCH_ENDPOINTS = ['tweets/search/recent', 'tweets/search/all', 'tweets/counts/all'] + pagination_token = 'next_token' if self.resource in SEARCH_ENDPOINTS else 'pagination_token' + if new_tweets: + self.params[pagination_token] = meta['previous_token'] + else: + self.params[pagination_token] = meta['next_token'] + + except TwitterRequestError as e: + if e.status_code < 500: + raise + continue + except TwitterConnectionError: + continue \ No newline at end of file diff --git a/TwitterAPI/TwitterRestPager.py b/TwitterAPI/TwitterRestPager.py deleted file mode 100644 index e7ce311..0000000 --- a/TwitterAPI/TwitterRestPager.py +++ /dev/null @@ -1,98 +0,0 @@ -__author__ = "geduldig" -__date__ = "June 8, 2013" -__license__ = "MIT" - - -from requests.exceptions import ConnectionError, ReadTimeout, SSLError -from requests.packages.urllib3.exceptions import ReadTimeoutError, ProtocolError -from .TwitterError import * -import requests -import time - - -class TwitterRestPager(object): - - """Continuous (stream-like) pagination of response from Twitter REST API resource. - - :param api: An authenticated TwitterAPI object - :param resource: String with the resource path (ex. search/tweets) - :param params: Dictionary of resource parameters - """ - - def __init__(self, api, resource, params=None): - self.api = api - self.resource = resource - self.params = params - - def get_iterator(self, wait=5, new_tweets=False): - """Iterate response from Twitter REST API resource. Resource is called - in a loop to retrieve consecutive pages of results. - - :param wait: Floating point number (default=5) of seconds wait between requests. - Depending on the resource, appropriate values are 5 or 60 seconds. - :param new_tweets: Boolean determining the search direction. - False (default) retrieves old results. - True retrieves current results. - - :returns: JSON objects containing statuses, errors or other return info. - :raises: TwitterRequestError - """ - elapsed = 0 - while True: - try: - # get one page of results - start = time.time() - r = self.api.request(self.resource, self.params) - it = r.get_iterator() - if new_tweets: - it = reversed(list(it)) - - # yield each item in the page - id = None - for item in it: - if 'id' in item: - id = item['id'] - if 'code' in item: - if item['code'] in [130, 131]: - # Twitter service error - raise TwitterConnectionError(item) - yield item - - # if a cursor is present, use it to get next page - # (otherwise, use id to get next page) - json = r.json() - cursor = -1 - if new_tweets and 'previous_cursor' in json: - cursor = json['previous_cursor'] - elif not new_tweets and 'next_cursor' in json: - cursor = json['next_cursor'] - - # bail when no more results - if cursor == 0: - break - elif cursor == -1 and not new_tweets and id is None: - break - - # sleep before getting another page of results - elapsed = time.time() - start - pause = wait - elapsed if elapsed < wait else 0 - time.sleep(pause) - - # waiting for new results to come in - if id is None: - continue - - # use either id or cursor to get a new page - if cursor != -1: - self.params['cursor'] = cursor - elif new_tweets: - self.params['since_id'] = str(id) - else: - self.params['max_id'] = str(id - 1) - - except TwitterRequestError as e: - if e.status_code < 500: - raise - continue - except TwitterConnectionError: - continue diff --git a/TwitterAPI/__init__.py b/TwitterAPI/__init__.py index cf4921b..ae9e26c 100644 --- a/TwitterAPI/__init__.py +++ b/TwitterAPI/__init__.py @@ -1,5 +1,5 @@ __title__ = 'TwitterAPI' -__version__ = '2.4.8' +__version__ = '2.8.2' __author__ = 'geduldig' __license__ = 'MIT' __copyright__ = 'Copyright 2013 geduldig' @@ -13,10 +13,10 @@ try: - from .TwitterAPI import TwitterAPI, TwitterResponse + from .TwitterAPI import TwitterAPI, TwitterResponse, OAuthType, HydrateType from .TwitterError import TwitterConnectionError, TwitterRequestError from .TwitterOAuth import TwitterOAuth - from .TwitterRestPager import TwitterRestPager + from .TwitterPager import TwitterPager except: pass @@ -26,5 +26,5 @@ 'TwitterConnectionError', 'TwitterRequestError', 'TwitterOAuth', - 'TwitterRestPager' + 'TwitterPager' ] diff --git a/TwitterAPI/constants.py b/TwitterAPI/constants.py index 69b755f..5c2c864 100644 --- a/TwitterAPI/constants.py +++ b/TwitterAPI/constants.py @@ -15,204 +15,285 @@ PROTOCOL = 'https' - DOMAIN = 'twitter.com' + VERSION = '1.1' CURATOR_VERSION = 'broadcast/1' -ADS_VERSION = '2' - -USER_AGENT = 'python-TwitterAPI' +ADS_VERSION = '11' -CONNECTION_TIMEOUT = 5 -STREAMING_TIMEOUT = 90 -REST_TIMEOUT = 5 ENDPOINTS = { - # resource: (method, subdomain) + # resource: (method(s), subdomain) # STREAMING API - 'statuses/filter': ('POST', 'stream'), - 'statuses/firehose': ('GET', 'stream'), - 'statuses/sample': ('GET', 'stream'), - 'site': ('GET', 'sitestream'), - 'user': ('GET', 'userstream'), + 'statuses/filter': ('POST', 'stream'), + 'statuses/firehose': ('GET', 'stream'), + 'statuses/sample': ('GET', 'stream'), + 'site': ('GET', 'sitestream'), + 'user': ('GET', 'userstream'), # PUBLIC API - 'account/remove_profile_banner': ('POST', 'api'), - 'account/settings': ('GET', 'api'), - 'account/update_delivery_device': ('POST', 'api'), - 'account/update_profile': ('POST', 'api'), - 'account/update_profile_background_image': ('POST', 'api'), - 'account/update_profile_banner': ('POST', 'api'), - 'account/update_profile_colors': ('POST', 'api'), - 'account/update_profile_image': ('POST', 'api'), - 'account/verify_credentials': ('GET', 'api'), + 'account/remove_profile_banner': ('POST', 'api'), + 'account/settings': ('GET', 'api'), + 'account/update_delivery_device': ('POST', 'api'), + 'account/update_profile': ('POST', 'api'), + 'account/update_profile_background_image': ('POST', 'api'), + 'account/update_profile_banner': ('POST', 'api'), + 'account/update_profile_colors': ('POST', 'api'), + 'account/update_profile_image': ('POST', 'api'), + 'account/verify_credentials': ('GET', 'api'), - 'application/rate_limit_status': ('GET', 'api'), - - 'blocks/create': ('POST', 'api'), - 'blocks/destroy': ('POST', 'api'), - 'blocks/ids': ('GET', 'api'), - 'blocks/list': ('GET', 'api'), - - 'direct_messages': ('GET', 'api'), - 'direct_messages/destroy': ('POST', 'api'), - 'direct_messages/events/new': ('POST', 'api'), - 'direct_messages/events/list': ('GET', 'api'), - 'direct_messages/events/show': ('GET', 'api'), - 'direct_messages/new': ('POST', 'api'), - 'direct_messages/sent': ('GET', 'api'), - 'direct_messages/show': ('GET', 'api'), - 'direct_messages/welcome_messages/new': ('POST', 'api'), - 'direct_messages/welcome_messages/show': ('GET', 'api'), - 'direct_messages/welcome_messages/destroy': ('DELETE', 'api'), - 'direct_messages/welcome_messages/rules/new': ('POST', 'api'), - 'direct_messages/welcome_messages/rules/list': ('GET', 'api'), - 'direct_messages/welcome_messages/rules/show': ('GET', 'api'), - 'direct_messages/welcome_messages/rules/destroy': ('DELETE', 'api'), - - 'favorites/create': ('POST', 'api'), - 'favorites/destroy': ('POST', 'api'), - 'favorites/list': ('GET', 'api'), - - 'followers/ids': ('GET', 'api'), - 'followers/list': ('GET', 'api'), - - 'friends/ids': ('GET', 'api'), - 'friends/list': ('GET', 'api'), - - 'friendships/create': ('POST', 'api'), - 'friendships/destroy': ('POST', 'api'), - 'friendships/incoming': ('GET', 'api'), - 'friendships/lookup': ('GET', 'api'), - 'friendships/no_retweets/ids': ('GET', 'api'), - 'friendships/outgoing': ('GET', 'api'), - 'friendships/show': ('GET', 'api'), - 'friendships/update': ('POST', 'api'), - - 'lists/create': ('POST', 'api'), - 'lists/destroy': ('POST', 'api'), - 'lists/list': ('GET', 'api'), - 'lists/members': ('GET', 'api'), - 'lists/members/create': ('POST', 'api'), - 'lists/members/create_all': ('POST', 'api'), - 'lists/members/destroy': ('POST', 'api'), - 'lists/members/destroy_all': ('POST', 'api'), - 'lists/members/show': ('GET', 'api'), - 'lists/memberships': ('GET', 'api'), - 'lists/ownerships': ('GET', 'api'), - 'lists/show': ('GET', 'api'), - 'lists/statuses': ('GET', 'api'), - 'lists/subscribers': ('GET', 'api'), - 'lists/subscribers/create': ('POST', 'api'), - 'lists/subscribers/destroy': ('POST', 'api'), - 'lists/subscribers/show': ('GET', 'api'), - 'lists/subscriptions': ('GET', 'api'), - 'lists/update': ('POST', 'api'), - - 'media/upload': ('POST', 'upload'), - - 'mutes/users/create': ('POST', 'api'), - 'mutes/users/destroy': ('POST', 'api'), - 'mutes/users/ids': ('GET', 'api'), - 'mutes/users/list': ('GET', 'api'), - - 'geo/id/:PARAM': ('GET', 'api'), # PLACE_ID - 'geo/place': ('POST', 'api'), - 'geo/reverse_geocode': ('GET', 'api'), - 'geo/search': ('GET', 'api'), - 'geo/similar_places': ('GET', 'api'), - - 'help/configuration': ('GET', 'api'), - 'help/languages': ('GET', 'api'), - 'help/privacy': ('GET', 'api'), - 'help/tos': ('GET', 'api'), - - 'saved_searches/create': ('POST', 'api'), - 'saved_searches/destroy/:PARAM': ('POST', 'api'), # ID - 'saved_searches/list': ('GET', 'api'), - 'saved_searches/show/:PARAM': ('GET', 'api'), # ID - - 'search/tweets': ('GET', 'api'), - - 'statuses/destroy/:PARAM': ('POST', 'api'), # ID - 'statuses/home_timeline': ('GET', 'api'), - 'statuses/lookup': ('GET', 'api'), - 'statuses/mentions_timeline': ('GET', 'api'), - 'statuses/oembed': ('GET', 'api'), - 'statuses/retweet/:PARAM': ('POST', 'api'), # ID - 'statuses/retweeters/ids': ('GET', 'api'), - 'statuses/retweets/:PARAM': ('GET', 'api'), # ID - 'statuses/retweets_of_me': ('GET', 'api'), - 'statuses/show/:PARAM': ('GET', 'api'), # ID - 'statuses/unretweet/:PARAM': ('POST', 'api'), # ID - 'statuses/user_timeline': ('GET', 'api'), - 'statuses/update': ('POST', 'api'), # deprecated - 'statuses/update_with_media': ('POST', 'api'), - - 'trends/available': ('GET', 'api'), - 'trends/closest': ('GET', 'api'), - 'trends/place': ('GET', 'api'), - - 'users/contributees': ('GET', 'api'), - 'users/contributors': ('GET', 'api'), - 'users/lookup': ('POST', 'api'), - 'users/profile_banner': ('GET' 'api'), - 'users/report_spam': ('POST', 'api'), - 'users/search': ('GET', 'api'), - 'users/show': ('GET', 'api'), - 'users/suggestions': ('GET', 'api'), - 'users/suggestions/:PARAM': ('GET', 'api'), # SLUG - 'users/suggestions/:PARAM/members': ('GET', 'api'), # SLUG + 'application/rate_limit_status': ('GET', 'api'), + + 'blocks/create': ('POST', 'api'), + 'blocks/destroy': ('POST', 'api'), + 'blocks/ids': ('GET', 'api'), + 'blocks/list': ('GET', 'api'), + + 'direct_messages': ('GET', 'api'), # deprecated + 'direct_messages/destroy': ('POST', 'api'), # deprecated + 'direct_messages/events/destroy': ('DELETE', 'api'), + 'direct_messages/events/new': ('POST', 'api'), + 'direct_messages/events/list': ('GET', 'api'), + 'direct_messages/events/show': ('GET', 'api'), + 'direct_messages/indicate_typing': ('POST', 'api'), + 'direct_messages/new': ('POST', 'api'), # deprecated + 'direct_messages/sent': ('GET', 'api'), # deprecated + 'direct_messages/show': ('GET', 'api'), # deprecated + 'direct_messages/welcome_messages/new': ('POST', 'api'), + 'direct_messages/welcome_messages/list': ('GET', 'api'), + 'direct_messages/welcome_messages/show': ('GET', 'api'), + 'direct_messages/welcome_messages/destroy': ('DELETE', 'api'), + 'direct_messages/welcome_messages/rules/new': ('POST', 'api'), + 'direct_messages/welcome_messages/rules/list': ('GET', 'api'), + 'direct_messages/welcome_messages/rules/show': ('GET', 'api'), + 'direct_messages/welcome_messages/rules/destroy': ('DELETE', 'api'), + + 'favorites/create': ('POST', 'api'), + 'favorites/destroy': ('POST', 'api'), + 'favorites/list': ('GET', 'api'), + + 'followers/ids': ('GET', 'api'), + 'followers/list': ('GET', 'api'), + + 'friends/ids': ('GET', 'api'), + 'friends/list': ('GET', 'api'), + + 'friendships/create': ('POST', 'api'), + 'friendships/destroy': ('POST', 'api'), + 'friendships/incoming': ('GET', 'api'), + 'friendships/lookup': ('GET', 'api'), + 'friendships/no_retweets/ids': ('GET', 'api'), + 'friendships/outgoing': ('GET', 'api'), + 'friendships/show': ('GET', 'api'), + 'friendships/update': ('POST', 'api'), + + 'lists/create': ('POST', 'api'), + 'lists/destroy': ('POST', 'api'), + 'lists/list': ('GET', 'api'), + 'lists/members': ('GET', 'api'), + 'lists/members/create': ('POST', 'api'), + 'lists/members/create_all': ('POST', 'api'), + 'lists/members/destroy': ('POST', 'api'), + 'lists/members/destroy_all': ('POST', 'api'), + 'lists/members/show': ('GET', 'api'), + 'lists/memberships': ('GET', 'api'), + 'lists/ownerships': ('GET', 'api'), + 'lists/show': ('GET', 'api'), + 'lists/statuses': ('GET', 'api'), + 'lists/subscribers': ('GET', 'api'), + 'lists/subscribers/create': ('POST', 'api'), + 'lists/subscribers/destroy': ('POST', 'api'), + 'lists/subscribers/show': ('GET', 'api'), + 'lists/subscriptions': ('GET', 'api'), + 'lists/update': ('POST', 'api'), + + 'media/metadata/create': ('POST', 'upload'), + 'media/upload': (['POST','GET'], 'upload'), + 'media/subtitles/create': ('POST', 'upload'), + 'media/subtitles/delete': ('POST', 'upload'), + + 'mutes/users/create': ('POST', 'api'), + 'mutes/users/destroy': ('POST', 'api'), + 'mutes/users/ids': ('GET', 'api'), + 'mutes/users/list': ('GET', 'api'), + + 'geo/id/:PARAM': ('GET', 'api'), # PLACE_ID + 'geo/place': ('POST', 'api'), + 'geo/reverse_geocode': ('GET', 'api'), + 'geo/search': ('GET', 'api'), + 'geo/similar_places': ('GET', 'api'), + + 'help/configuration': ('GET', 'api'), + 'help/languages': ('GET', 'api'), + 'help/privacy': ('GET', 'api'), + 'help/tos': ('GET', 'api'), + + 'saved_searches/create': ('POST', 'api'), + 'saved_searches/destroy/:PARAM': ('POST', 'api'), # ID + 'saved_searches/list': ('GET', 'api'), + 'saved_searches/show/:PARAM': ('GET', 'api'), # ID + + 'search/tweets': ('GET', 'api'), + + 'statuses/destroy/:PARAM': ('POST', 'api'), # ID + 'statuses/home_timeline': ('GET', 'api'), + 'statuses/lookup': ('GET', 'api'), + 'statuses/mentions_timeline': ('GET', 'api'), + 'statuses/oembed': ('GET', 'api'), + 'statuses/retweet/:PARAM': ('POST', 'api'), # ID + 'statuses/retweeters/ids': ('GET', 'api'), + 'statuses/retweets/:PARAM': ('GET', 'api'), # ID + 'statuses/retweets_of_me': ('GET', 'api'), + 'statuses/show/:PARAM': ('GET', 'api'), # ID + 'statuses/unretweet/:PARAM': ('POST', 'api'), # ID + 'statuses/user_timeline': ('GET', 'api'), + 'statuses/update': ('POST', 'api'), + 'statuses/update_with_media': ('POST', 'api'), # deprecated + + 'trends/available': ('GET', 'api'), + 'trends/closest': ('GET', 'api'), + 'trends/place': ('GET', 'api'), + + 'users/contributees': ('GET', 'api'), + 'users/contributors': ('GET', 'api'), + 'users/lookup': ('POST', 'api'), + 'users/profile_banner': ('GET' 'api'), + 'users/report_spam': ('POST', 'api'), + 'users/search': ('GET', 'api'), + 'users/show': ('GET', 'api'), + 'users/suggestions': ('GET', 'api'), + 'users/suggestions/:PARAM': ('GET', 'api'), # SLUG + 'users/suggestions/:PARAM/members': ('GET', 'api'), # SLUG # COLLECTIONS API - 'collections/create': ('POST', 'api'), - 'collections/destroy': ('POST', 'api'), - 'collections/entries': ('GET', 'api'), - 'collections/entries/add': ('POST', 'api'), - 'collections/entries/curate': ('POST', 'api'), - 'collections/entries/move': ('POST', 'api'), - 'collections/entries/remove': ('POST', 'api'), - 'collections/list': ('GET', 'api'), - 'collections/show': ('GET', 'api'), - 'collections/update': ('POST', 'api'), + 'collections/create': ('POST', 'api'), + 'collections/destroy': ('POST', 'api'), + 'collections/entries': ('GET', 'api'), + 'collections/entries/add': ('POST', 'api'), + 'collections/entries/curate': ('POST', 'api'), + 'collections/entries/move': ('POST', 'api'), + 'collections/entries/remove': ('POST', 'api'), + 'collections/list': ('GET', 'api'), + 'collections/show': ('GET', 'api'), + 'collections/update': ('POST', 'api'), # CURATOR API - 'collections/:PARAM/content': ('GET', 'curator'), # ID - 'projects': ('GET', 'curator'), - 'projects/:PARAM': ('GET', 'curator'), # ID - 'streams/:PARAM/content': ('GET', 'curator'), # ID - 'streams/:PARAM/metrics': ('GET', 'curator'), # ID - 'streams/:PARAM/trendline': ('GET', 'curator'), # ID - 'streams/compare': ('GET', 'curator'), - 'streams/compare_to_target': ('GET', 'curator'), + 'collections/:PARAM/content': ('GET', 'curator'), # ID + 'projects': ('GET', 'curator'), + 'projects/:PARAM': ('GET', 'curator'), # ID + 'streams/:PARAM/content': ('GET', 'curator'), # ID + 'streams/:PARAM/metrics': ('GET', 'curator'), # ID + 'streams/:PARAM/trendline': ('GET', 'curator'), # ID + 'streams/compare': ('GET', 'curator'), + 'streams/compare_to_target': ('GET', 'curator'), # ADS API (not tested!!) - 'accounts/:PARAM/auction_insights': ('GET', 'ads-api'), # ACCOUNT ID - 'stats/accounts/:PARAM': ('GET', 'ads-api'), # ACCOUNT ID - 'stats/accounts/:PARAM/reach/funding_instruments': ('GET', 'ads-api'), # ACCOUNT ID - 'stats/jobs/accounts/:PARAM': ('GET', 'ads-api'), # ACCOUNT ID - 'stats/jobs/accounts/:PARAM': ('POST', 'ads-api'), # ACCOUNT ID - 'stats/jobs/accounts/:PARAM/:PARAM': ('DELETE', 'ads-api'), # ACCOUNT ID, JOB ID - 'stats/jobs/summaries': ('GET', 'ads-api'), + 'accounts/:PARAM/auction_insights': ('GET', 'ads-api'), # ACCOUNT ID + 'accounts/:PARAM/promoted_tweets': ('GET', 'ads-api'), # ACCOUNT ID + 'stats/accounts/:PARAM': ('GET', 'ads-api'), # ACCOUNT ID + 'stats/accounts/:PARAM/reach/funding_instruments': ('GET', 'ads-api'), # ACCOUNT ID + 'stats/jobs/accounts/:PARAM': ('GET', 'ads-api'), # ACCOUNT ID + 'stats/jobs/accounts/:PARAM': ('POST', 'ads-api'), # ACCOUNT ID + 'stats/jobs/accounts/:PARAM/:PARAM': ('DELETE', 'ads-api'), # ACCOUNT ID, JOB ID + 'stats/jobs/summaries': ('GET', 'ads-api'), + + # ACCOUNT ACTIVITY WEBHOOK API + + 'account_activity/all/:PARAM/subscriptions': (['POST', 'DELETE'], 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions/all': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions/all/list': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/webhooks': ('POST', 'api'), # ENVIRONMENT NAME + 'account_activity/all/:PARAM/webhooks/:PARAM': ('DELETE', 'api'), # ENVIRONMENT NAME, WEBHOOK ID + 'account_activity/all/count': ('GET', 'api'), + 'account_activity/all/webhooks': ('GET', 'api'), + 'account_activity/webhooks': ('POST', 'api'), + 'account_activity/webhooks/:PARAM': ('DELETE', 'api'), # WEBHOOK ID + 'account_activity/webhooks/:PARAM/subscriptions': ('POST', 'api'), # ENVIRONMENT NAME + 'account_activity/webhooks/:PARAM/subscriptions/list': ('GET', 'api'), # ENVIRONMENT NAME + + # PREMIUM SEARCH API + + 'tweets/search/30day/:PARAM': ('GET', 'api'), # LABEL + 'tweets/search/30day/:PARAM/counts': ('GET', 'api'), # LABEL + 'tweets/search/fullarchive/:PARAM': ('GET', 'api'), # LABEL + 'tweets/search/fullarchive/:PARAM/counts': ('GET', 'api'), # LABEL + + # LABS API (BETAS) WILL NEED APPLICATION APPROVAL + + 'labs/1/tweets/metrics/private': ('GET', 'api'), + 'labs/2/tweets/:PARAM': ('GET', 'api'), # TWEET ID + 'labs/2/tweets': ('GET', 'api'), + 'labs/2/tweets/search': ('GET', 'api'), + 'labs/2/tweets/:PARAM/hidden': ('PUT', 'api'), # TWEET ID + 'labs/2/users/:PARAM': ('GET', 'api'), # USER ID + 'labs/2/users': ('GET', 'api'), - # NEW ACCOUNT ACTIVITY WEBHOOK API (BETA) - - 'account_activity/all/:PARAM/webhooks': ('POST', 'api'), - 'account_activity/all/:PARAM/subscriptions': ('POST', 'api'), - 'account_activity/all/:PARAM/subscriptions/all': ('GET', 'api'), - 'account_activity/all/:PARAM/subscriptions/all/list': ('GET', 'api'), - 'account_activity/all/:PARAM/subscriptions': ('DELETE', 'api'), - 'account_activity/all/count': ('GET', 'api'), - 'account_activity/all/webhooks': ('GET', 'api'), - 'account_activity/webhooks': ('POST', 'api'), - 'account_activity/webhooks/:PARAM': ('DELETE', 'api'), - 'account_activity/webhooks/:PARAM/subscriptions': ('POST', 'api'), - 'account_activity/webhooks/:PARAM/subscriptions/list': ('GET', 'api') + # !!!!!!!!!!!!!!!!!!! + # API V2 EARLY ACCESS + # !!!!!!!!!!!!!!!!!!! + + 'compliance/jobs': (['GET','POST'], 'api'), + 'compliance/jobs/:PARAM': ('GET', 'api'), # ID + + 'lists': ('POST', 'api'), + 'lists/:PARAM': (['GET','PUT','DELETE'], 'api'), # ID + 'lists/:PARAM/followers': ('GET', 'api'), + 'lists/:PARAM/members': (['GET','POST'], 'api'), # ID + 'lists/:PARAM/members/:PARAM': ('DELETE', 'api'), # ID, USER ID + 'lists/:PARAM/tweets': ('GET', 'api'), # ID + + 'spaces': ('GET', 'api'), + 'spaces/:PARAM': ('GET', 'api'), # ID + 'spaces/:PARAM/buyers': ('GET', 'api'), # ID + 'spaces/by/creator_ids': ('GET', 'api'), + 'spaces/search': ('GET', 'api'), + + 'tweets': (['GET','POST'], 'api'), + 'tweets/:PARAM': (['GET','DELETE'], 'api'), # ID + 'tweets/:PARAM/hidden': ('PUT', 'api'), # ID + 'tweets/:PARAM/liking_users': ('GET', 'api'), # ID + 'tweets/:PARAM/retweeted_by': ('GET', 'api'), # ID + 'tweets/counts/all': ('GET', 'api'), + 'tweets/counts/recent': ('GET', 'api'), + 'tweets/sample/stream': ('GET', 'api'), + 'tweets/search/all': ('GET', 'api'), + 'tweets/search/recent': ('GET', 'api'), + 'tweets/search/stream': ('GET', 'api'), + 'tweets/search/stream/rules': (['POST','GET'], 'api'), + + 'users': ('GET', 'api'), + 'users/:PARAM': ('GET', 'api'), # ID + 'users/:PARAM/blocking': (['GET','POST'], 'api'), # ID + 'users/:PARAM/bookmarks': (['GET','DELETE'], 'api'), # ID + 'users/:PARAM/followed_lists': (['GET','POST'], 'api'), # ID + 'users/:PARAM/followed_lists/:PARAM': ('DELETE', 'api'), # ID, LIST ID + 'users/:PARAM/followers': ('GET', 'api'), # ID + 'users/:PARAM/following': (['GET','POST'], 'api'), # ID + 'users/:PARAM/following/:PARAM': ('DELETE', 'api'), # USER SOURCE ID, USER TARGET ID + 'users/:PARAM/liked_tweets': ('GET', 'api'), # ID + 'users/:PARAM/likes': ('POST', 'api'), # ID + 'users/:PARAM/likes/:PARAM': ('DELETE', 'api'), # USER ID, TWEET ID + 'users/:PARAM/liked_tweets': ('GET', 'api'), # ID + 'users/:PARAM/list_memberships': ('GET', 'api'), # ID + 'users/:PARAM/mentions': ('GET', 'api'), # ID + 'users/:PARAM/muting': ('GET', 'api'), # ID + 'users/:PARAM/owned_lists': ('GET', 'api'), # ID + 'users/:PARAM/pinned_lists': (['GET','POST'], 'api'), # ID + 'users/:PARAM/pinned_lists/:PARAM': ('DELETE', 'api'), # ID, LIST ID + 'users/:PARAM/retweets': ('POST', 'api'), # ID + 'users/:PARAM/retweets/:PARAM': ('DELETE', 'api'), # ID, SOURCE TWEET ID + 'users/:PARAM/tweets': ('GET', 'api'), # ID + 'users/:PARAM/blocking/:PARAM': ('DELETE', 'api'), # SOURCE USER ID, TARGET USER ID + 'users/:PARAM/following/:PARAM': ('DELETE', 'api'), # SOURCE USER ID, TARGET USER ID + 'users/:PARAM/muting/:PARAM': ('DELETE', 'api'), # SOURCE USER ID, TARGET USER ID + 'users/by': ('GET', 'api'), + 'users/by/username/:PARAM': ('GET', 'api'), # USERNAME } diff --git a/docs/README.rst b/docs/README.rst new file mode 100644 index 0000000..07bd43e --- /dev/null +++ b/docs/README.rst @@ -0,0 +1,37 @@ +Instructions for Generating Documentation with Sphinx +===================================================== + +INSTALL SPHINX:: + + pip3 install -U Sphinx + +MAKE NEW DOCS IN MASTER BRANCH:: + + git clone git@github.com:geduldig/TwitterAPI.git + cd TwitterAPI/docs + vi conf.py (modify version and release) + make html + +TEST DOC WEB PAGE:: + + TwitterAPI/docs/_build/html/index.html + +DOWNLOAD OLD DOCS FROM GH-PAGES BRANCH:: + + git clone -b gh-pages git@github.com:geduldig/TwitterAPI.git + +COPY NEW DOCS OVER OLD DOCS:: + + cp ./_build/html/*.* ./TwitterAPI + cp -r ./_build/html/* ./TwitterAPI + rm -r ./_build + +UPLOAD NEW DOCS TO GH-PAGES BRANCH::: + + cd TwitterAPI + git commit -am "2.5 docs" + git push origin gh-pages + +CLEAN UP:: + cd .. + rm -r TwitterAPI \ No newline at end of file diff --git a/docs/authentication.rst b/docs/authentication.rst index c9f6ad7..8af0979 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1,9 +1,9 @@ Authentication ============== -The first thing you need to do is create an application on `apps.twitter.com `_ and generate your oAuth keys. +Before you can do anything you must create an application on `apps.twitter.com `_ and generate oAuth keys. -Twitter supports both user and application authentication, known as oAuth 1 and oAuth 2, respectively. User authentication gives you access to all API endpoints, basically read and write persmission. Application authentication gives you access to just the read portion of the API -- so, no creating or destroying tweets. Application authentication also has elevated rate limits. +Twitter supports both user and application authentication, called oAuth 1 and oAuth 2, respectively. User authentication gives you access to all API endpoints, basically read and write persmission. It is also required in order to using the Streaming API. Application authentication gives you access to just the read portion of the API -- so, no creating or destroying tweets. Application authentication, however, has elevated rate limits. User Authentication ------------------- diff --git a/docs/conf.py b/docs/conf.py index 66e5e54..2819b98 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,6 +30,9 @@ # ones. extensions = [ 'sphinx.ext.autodoc', + "sphinx.ext.intersphinx", + "sphinx.ext.mathjax", + "sphinx.ext.viewcode", ] # Add any paths that contain templates here, relative to this directory. @@ -46,16 +49,16 @@ # General information about the project. project = u'TwitterAPI' -copyright = u'2014, geduldig' +copyright = u'2021, geduldig' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '2.3' +version = '2.7' # The full version, including alpha/beta/rc tags. -release = '2.3' +release = '2.7' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/errors.rst b/docs/errors.rst index 9d90d21..04471c0 100644 --- a/docs/errors.rst +++ b/docs/errors.rst @@ -1,7 +1,7 @@ Error Handling ============== -Besides tweet statuses, the REST API and Streaming API iterators may return error and other messages. It is up to the application to test what type of object has been returned. Message types are documented `here `_ and `here `_. +Besides tweet statuses, the REST API and Streaming API iterators can return errors and other messages. It is up to the application to test what type of object has been returned. Message types are documented `here `_ and `here `_. REST API Messages ----------------- @@ -20,7 +20,7 @@ REST API endpoints can return many more types of messages than Streaming API end Streaming API Messages ---------------------- -Streaming API endpoints return a variety of messages, most are not really errors. For example, a "limit" message contains the number of tweets missing from the stream. This happens when the number of tweets matching your filter exceeds a threshold set by Twitter. Other useful messages are "disconnect" and "delete". The pattern is similar to the one preceding: +Streaming API endpoints return a variety of messages. Some are not errors. For example, a "limit" message contains the number of tweets missing from the stream. This happens when the number of tweets matching your filter exceeds a threshold set by Twitter. Other useful messages are "disconnect" and "delete". .. code-block:: python @@ -34,4 +34,4 @@ Streaming API endpoints return a variety of messages, most are not really errors print 'disconnecting because %s' % item['disconnect']['reason'] break -Even if you are not interested in handling errors it is necessary to test that the object returned by an iterator is a valid tweet status before using the object. \ No newline at end of file +Even if you are not interested in handling errors it is necessary to test that the object returned by an iterator is a valid tweet status before using the object. Valid tweet objects have a 'text' property (or a 'full_text' property if it is an extended tweet). \ No newline at end of file diff --git a/docs/examples.rst b/docs/examples.rst index 723fb04..93d44af 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -26,7 +26,7 @@ Get your entire timeline .. code-block:: python - pager = TwitterRestPager(api, 'statuses/home_timeline', {'count':200}) + pager = TwitterPager(api, 'statuses/home_timeline', {'count':200}) for item in pager.get_iterator(wait=60): if 'text' in item: print item['text'] diff --git a/docs/faulttolerance.rst b/docs/faulttolerance.rst index 3bbc650..f8059a3 100644 --- a/docs/faulttolerance.rst +++ b/docs/faulttolerance.rst @@ -46,17 +46,16 @@ Last Week's Pages Requests for REST API endpoints can throw `TwitterRequestError <./twittererror.html>`_ and `TwitterConnectionError <./twittererror.html>`_. They do not, however, return "disconnect" messages. Twitter returns error messages for these endpoints with "message". Most of these errors require attention before re-trying the request, except those with codes of 130 or 131, which are internal server errors. -For making continuos REST API requests (i.e. paging), TwitterAPI provides `TwitterRestPager <./paging.html>`_. If you use this class to request tweets that have been posted back to one week old, for example, the class's iterator will handle both types of exceptions automatically. The iterator also handles "message" objects with 130 or 131 codes for you. Any other "message" object gets passed on for you to handle. +For making continuos REST API requests (i.e. paging), TwitterAPI provides `TwitterPager <./paging.html>`_. If you use this class to request tweets that have been posted back to one week old, for example, the class's iterator will handle both types of exceptions automatically. The iterator also handles "message" objects with 130 or 131 codes for you. Any other "message" object gets passed on for you to handle. One final consideration is the endpoint's rate limit, determinted by the endpoint and whether you authenticate with oAuth 1 or oAuth 2. By default, the iterator waits 5 seconds between requests. This is sufficient for 180 requests in 15 minutes, the rate limit for "search/tweets" with oAuth 1. You can do better with oAuth 2. It permits 450 requests every 15 minutes, or 1 request per 2 seconds. The example below sets the wait assuming oAuth 2 rate limits. .. code-block:: python - iterator = TwitterRestPager(api, 'search/tweets', {'q':'pizza'}).get_iterator(wait=2) - try: - for item in iterator: - if 'text' in item: - print(item['text']) - elif 'message' in item: - # something needs to be fixed before re-connecting - raise Exception(item['message']) + iterator = TwitterPager(api, 'search/tweets', {'q':'pizza'}).get_iterator(wait=2) + for item in iterator: + if 'text' in item: + print(item['text']) + elif 'message' in item: + # something needs to be fixed before re-connecting + raise Exception(item['message']) \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 80112db..439c4aa 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,7 +3,7 @@ Introduction *Minimal Python wrapper for Twitter's REST and Streaming APIs* -The principle behind TwitterAPI's design is to provide a single method for accessing the Twitter API. You can call the ``request`` method with *any* endpoint found on Twitter's `developer site `_, the complete reference for all endpoints. The benefits of a single-method approach are: less code for me to maintain, and just a single method for you to learn. Here is a quck example: +The principle behind TwitterAPI's design is to provide a single method for accessing the Twitter API. The ``request`` method allows one to call *any* endpoint found on Twitter's `developer site `_, the complete reference for all endpoints. The benefits of a single-method approach are: 1) less code for me to maintain, and 2) a single method for you to learn. Here is a quck example: .. code-block:: python @@ -12,16 +12,16 @@ The principle behind TwitterAPI's design is to provide a single method for acces r = api.request('search/tweets', {'q':'pizza'}) print r.status_code -If you want Twitter's entire response as one long string, containing tweets in this example, you would use ``r.text``. But, often an iterator is more useful: +Get Twitter's entire response as one long string containing tweets (in this example) by calling ``r.text``. More often an iterator is useful: .. code-block:: python for item in r.get_iterator(): print item['user']['screen_name'], item['text'] -The iterator returns decoded JSON objects. What makes the iterator very powerful is it works with both REST API and Streaming API endpoints. No syntax changes required; just supply any endpoint and parameters that are found on Twitter's dev site. +The iterator returns JSON objects. What makes the iterator very powerful is it works with both REST API and Streaming API endpoints. No syntax changes required -- supply any endpoint and parameters that are found on Twitter's developer site. -TwitterAPI is compatible with Python 2 and Python 3. It authenticates using either OAauth 1 or OAuth 2. It also supports web proxy server authentication. All with very little code change for you. +TwitterAPI is compatible with Python 2 and Python 3. It authenticates using either OAauth 1 or OAuth 2. It also supports web proxy server authentication. All this with minimal code change for you. Topics ====== @@ -30,10 +30,10 @@ Topics :maxdepth: 1 authentication.rst - errors.rst paging.rst - examples.rst + errors.rst faulttolerance.rst + examples.rst Modules ======= @@ -49,5 +49,5 @@ Optional: .. toctree:: :maxdepth: 2 - twitterrestpager.rst + twitterpager.rst twitteroauth.rst diff --git a/docs/paging.rst b/docs/paging.rst index 5e9f6bd..7e53e47 100644 --- a/docs/paging.rst +++ b/docs/paging.rst @@ -1,16 +1,16 @@ Paging Results ============== -Paging refers to getting successive batches of results. The Streaming API endpoints in a sense do this inherently. REST API endpoints will never return more than a specificied maximum number of results. When you request `search/tweets`, for example, by default you will get at most 20 tweets. You can increase that number to a maximum of 100. This is the page size. Twitter provides a `way `_ to get successive pages, so it is possible to get more than 100 tweets with `search/tweets`, just not in a single request. - -If you don't want to implement paging yourself, you can use the `TwitterRestPager <./twitterrestpager.html>`_ helper class with any REST API endpoint that returns multiples of something. The following, for example, searches for all tweets containing 'pizza' that Twitter has stored -- about a week's worth. +Whether one is searching for tweets with `search/tweets` or downloading a user's timeline with `statuses/user_timeline` Twitter limits the number of tweets. So, in order to get more tweets, one must make successive requests and with each request skip the previously acquired tweets. This is done by specifying the tweet id from where to start. Twitter has a description `here `_. If you don't want to implement paging yourself, you can use the `TwitterPager <./twitterpager.html>`_ helper class with any REST API endpoint that returns multiples of something. The following, for example, searches for all tweets containing 'pizza' that Twitter has stored -- up to about a week's worth maximum. .. code-block:: python - r = TwitterRestPager(api, 'search/tweets', {'q':'pizza', 'count':100}) + r = TwitterPager(api, 'search/tweets', {'q':'pizza', 'count':100}) for item in r.get_iterator(): if 'text' in item: print item['text'] elif 'message' in item and item['code'] == 88: print 'SUSPEND, RATE LIMIT EXCEEDED: %s\n' % item['message'] break + +By default there is a built-in wait time of 5 seconds between successive calls. This value is overridden with an argument to `get_iterator()`. See the documentation also to learn how to wait for new tweets. In other words, the iterator can be setup to poll for newer pages of tweets rather than older pages. \ No newline at end of file diff --git a/docs/twitterpager.rst b/docs/twitterpager.rst new file mode 100644 index 0000000..ffd6103 --- /dev/null +++ b/docs/twitterpager.rst @@ -0,0 +1,6 @@ +TwitterAPI.TwitterPager +=========================== + +.. automodule:: TwitterAPI.TwitterPager + :members: + :undoc-members: diff --git a/docs/twitterrestpager.rst b/docs/twitterrestpager.rst deleted file mode 100644 index 2fe221c..0000000 --- a/docs/twitterrestpager.rst +++ /dev/null @@ -1,6 +0,0 @@ -TwitterAPI.TwitterRestPager -=========================== - -.. automodule:: TwitterAPI.TwitterRestPager - :members: - :undoc-members: diff --git a/examples/api_test.py b/examples/api_test.py deleted file mode 100644 index bbac44b..0000000 --- a/examples/api_test.py +++ /dev/null @@ -1,65 +0,0 @@ -import codecs -from datetime import datetime -import sys -from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRestPager - - -try: - # python 3 - sys.stdout = codecs.getwriter('utf8')(sys.stdout.buffer) -except: - # python 2 - sys.stdout = codecs.getwriter('utf8')(sys.stdout) - - -# SAVE YOUR APPLICATION CREDENTIALS IN TwitterAPI/credentials.txt. -o = TwitterOAuth.read_file() - -# Using OAuth1... -api = TwitterAPI(o.consumer_key, - o.consumer_secret, - o.access_token_key, - o.access_token_secret) - -# Using OAuth2... -#api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type="oAuth2") - - -TEST_NUMBER = 0 - - -try: - if TEST_NUMBER == 0: - - # VERIFY YOUR CREDS - r = api.request('account/verify_credentials') - print(r.text) - - if TEST_NUMBER == 1: - - # POST A TWEET - r = api.request('statuses/update', - {'status': 'the time is now %s' % datetime.now()}) - print(r.status_code) - - if TEST_NUMBER == 2: - - # GET 5 TWEETS CONTAINING 'ZZZ' - for item in api.request('search/tweets', {'q': 'zzz', 'count': 5}): - print(item['text'] if 'text' in item else item) - - if TEST_NUMBER == 3: - - # STREAM TWEETS FROM AROUND NYC - for item in api.request('statuses/filter', {'locations': '-74,40,-73,41'}): - print(item['text'] if 'text' in item else item) - - if TEST_NUMBER == 4: - - # GET TWEETS FROM THE PAST WEEK OR SO CONTAINING 'LOVE' - pager = TwitterRestPager(api, 'search/tweets', {'q': 'love'}) - for item in pager.get_iterator(): - print(item['text'] if 'text' in item else item) - -except Exception as e: - print(e) diff --git a/examples/cli/cli.py b/examples/cli/cli.py index 4ad8d0b..d3d9245 100644 --- a/examples/cli/cli.py +++ b/examples/cli/cli.py @@ -29,12 +29,12 @@ """ -__author__ = "geduldig" +__author__ = "Jonas Geduldig" __date__ = "June 7, 2013" __license__ = "MIT" -from TwitterAPI import TwitterAPI, __version__ +from TwitterAPI import TwitterAPI, TwitterRequestError, TwitterConnectionError, __version__ from TwitterAPI.TwitterOAuth import TwitterOAuth import argparse import codecs @@ -42,6 +42,9 @@ import sys +DEFAULT_API_VERSION = '1.1' + + # print UTF-8 to the console if sys.platform == "win32": from Unicode_win32 import stdout @@ -75,7 +78,7 @@ def _to_dict(param_list): if __name__ == '__main__': - print('TwitterAPI %s by geduldig' % __version__) + print('TwitterAPI %s by Jonas Geduldig' % __version__) parser = argparse.ArgumentParser( description='Request any Twitter Streaming or REST API endpoint') @@ -84,6 +87,13 @@ def _to_dict(param_list): metavar='FILENAME', type=str, help='file containing OAuth credentials') + parser.add_argument( + '-bearertoken', + metavar='BEARERTOKEN', + type=eval, + choices=[True, False], + default=False, + help='use OAuth 2.0') parser.add_argument( '-endpoint', metavar='ENDPOINT', @@ -108,27 +118,50 @@ def _to_dict(param_list): type=int, help='number of spaces to indent json output', default=None) + parser.add_argument( + '-version', + metavar='VERSION', + type=str, + help='Twitter API version', + default=DEFAULT_API_VERSION) + parser.add_argument( + '-methodoverride', + metavar='METHOD', + type=str, + help='override default HTTP method', + default=None) args = parser.parse_args() try: params = _to_dict(args.parameters) oauth = TwitterOAuth.read_file(args.oauth) + auth_type = 'oAuth2' if args.bearertoken else 'oAuth1' api = TwitterAPI(oauth.consumer_key, - oauth.consumer_secret, - oauth.access_token_key, - oauth.access_token_secret) - response = api.request(args.endpoint, params) + oauth.consumer_secret, + oauth.access_token_key, + oauth.access_token_secret, + api_version=args.version, + auth_type=auth_type) + response = api.request(args.endpoint, params, method_override=args.methodoverride) for item in response.get_iterator(): if not args.fields: - print( json.dumps(item, ensure_ascii='False', indent=args.indent)) + print(json.dumps(item, ensure_ascii='False', indent=args.indent)) else: for name in args.fields: value = _search(name, item) if value: print('%s: %s' % (name, value)) + except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + + except TwitterConnectionError as e: + print(e) + except KeyboardInterrupt: print('Terminated by user') diff --git a/examples/page_tweets.py b/examples/page_tweets.py deleted file mode 100644 index 4ae9f01..0000000 --- a/examples/page_tweets.py +++ /dev/null @@ -1,21 +0,0 @@ -from TwitterAPI import TwitterAPI, TwitterRestPager - - -SEARCH_TERM = 'pizza' - - -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN_KEY = '' -ACCESS_TOKEN_SECRET = '' - - -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET) - -pager = TwitterRestPager(api, 'search/tweets', {'q': SEARCH_TERM}) - -for item in pager.get_iterator(): - print(item['text'] if 'text' in item else item) diff --git a/examples/post_image.py b/examples/post_image.py deleted file mode 100644 index a71fc05..0000000 --- a/examples/post_image.py +++ /dev/null @@ -1,30 +0,0 @@ -''' -The method in this example for uploading image files is depricated. -See update_media.py for the recommeded way to tweet an image. -''' - -from TwitterAPI import TwitterAPI - - -TWEET_TEXT = 'some tweet text' -IMAGE_PATH = './some_image.png' - - -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN_KEY = '' -ACCESS_TOKEN_SECRET = '' - - -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET) - -file = open(IMAGE_PATH, 'rb') -data = file.read() -r = api.request('statuses/update_with_media', - {'status': TWEET_TEXT}, - {'media[]': data}) - -print('SUCCESS' if r.status_code == 200 else 'FAILURE') diff --git a/examples/post_tweet.py b/examples/post_tweet.py deleted file mode 100644 index 1956190..0000000 --- a/examples/post_tweet.py +++ /dev/null @@ -1,20 +0,0 @@ -from TwitterAPI import TwitterAPI - - -TWEET_TEXT = "Ce n'est pas un tweet tweet." - - -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN_KEY = '' -ACCESS_TOKEN_SECRET = '' - - -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET) - -r = api.request('statuses/update', {'status': TWEET_TEXT}) - -print('SUCCESS' if r.status_code == 200 else 'FAILURE') diff --git a/examples/search_tweets.py b/examples/search_tweets.py deleted file mode 100644 index adfc74f..0000000 --- a/examples/search_tweets.py +++ /dev/null @@ -1,23 +0,0 @@ -from TwitterAPI import TwitterAPI - - -SEARCH_TERM = 'pizza' - - -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN_KEY = '' -ACCESS_TOKEN_SECRET = '' - - -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET) - -r = api.request('search/tweets', {'q': SEARCH_TERM}) - -for item in r: - print(item['text'] if 'text' in item else item) - -print('\nQUOTA: %s' % r.get_rest_quota()) diff --git a/examples/stream_tweets.py b/examples/stream_tweets.py deleted file mode 100644 index ff2b0db..0000000 --- a/examples/stream_tweets.py +++ /dev/null @@ -1,21 +0,0 @@ -from TwitterAPI import TwitterAPI - - -TRACK_TERM = 'pizza' - - -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN_KEY = '' -ACCESS_TOKEN_SECRET = '' - - -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET) - -r = api.request('statuses/filter', {'track': TRACK_TERM}) - -for item in r: - print(item['text'] if 'text' in item else item) diff --git a/examples/update_media.py b/examples/update_media.py deleted file mode 100644 index e84d657..0000000 --- a/examples/update_media.py +++ /dev/null @@ -1,35 +0,0 @@ -''' -This example replaces post_image.py. It demonstrates a method for -posting an image, which now requires two API calls. -''' - -from TwitterAPI import TwitterAPI - - -TWEET_TEXT = 'some tweet text' -IMAGE_PATH = './some_image.png' - - -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN_KEY = '' -ACCESS_TOKEN_SECRET = '' - - -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET) - -# STEP 1 - upload image -file = open(IMAGE_PATH, 'rb') -data = file.read() -r = api.request('media/upload', None, {'media': data}) -print('UPLOAD MEDIA SUCCESS' if r.status_code == 200 else 'UPLOAD MEDIA FAILURE') - -# STEP 2 - post tweet with a reference to uploaded image -if r.status_code == 200: - media_id = r.json()['media_id'] - r = api.request( - 'statuses/update', {'status': TWEET_TEXT, 'media_ids': media_id}) - print('UPDATE STATUS SUCCESS' if r.status_code == 200 else 'UPDATE STATUS FAILURE') diff --git a/examples/oauth_test.py b/examples/v1.1/connect_using_login.py similarity index 91% rename from examples/oauth_test.py rename to examples/v1.1/connect_using_login.py index e694cc3..a3020d6 100644 --- a/examples/oauth_test.py +++ b/examples/v1.1/connect_using_login.py @@ -3,10 +3,8 @@ from urlparse import parse_qs from TwitterAPI import TwitterAPI - -consumer_key = '' -consumer_secret = '' - +consumer_key = +consumer_secret = # obtain request token oauth = OAuth1(consumer_key, consumer_secret) @@ -17,14 +15,12 @@ request_key = credentials.get('oauth_token')[0] request_secret = credentials.get('oauth_token_secret')[0] - # obtain authorization from resource owner print( 'Go here to authorize:\n https://api.twitter.com/oauth/authorize?oauth_token=%s' % request_key) verifier = raw_input('Enter your authorization code: ') - # obtain access token oauth = OAuth1(consumer_key, consumer_secret, @@ -36,11 +32,10 @@ access_token_key = credentials.get('oauth_token')[0] access_token_secret = credentials.get('oauth_token_secret')[0] - # access resource api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret) for item in api.request('statuses/filter', {'track': 'zzz'}): - print(item['text']) + print(item['text']) \ No newline at end of file diff --git a/examples/how_to_connect.py b/examples/v1.1/connect_using_oauth.py similarity index 59% rename from examples/how_to_connect.py rename to examples/v1.1/connect_using_oauth.py index c2b1606..46ef4cc 100644 --- a/examples/how_to_connect.py +++ b/examples/v1.1/connect_using_oauth.py @@ -1,31 +1,25 @@ from TwitterAPI import TwitterAPI - -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN_KEY = '' -ACCESS_TOKEN_SECRET = '' - - # If you are behind a firewall you may need to provide proxy server # authentication. proxy_url = None # Example: 'https://USERNAME:PASSWORD@PROXYSERVER:PORT' # Using OAuth 1.0 to authenticate you have access all Twitter endpoints. -# Using OAuth 2.0 to authenticate you lose access to user specific endpoints (ex. statuses/update), -# but you get higher rate limits. -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET, +api = TwitterAPI(, + , + , + , auth_type='oAuth1', proxy_url=proxy_url) -#api = TwitterAPI(CONSUMER_KEY, CONSUMER_SECRET, auth_type='oAuth2', proxy_url=proxy_url) - +# Using OAuth 2.0 to authenticate you lose access to user specific endpoints +# (ex. statuses/update and statuses/filter), but you increase your rate limits. +# api = TwitterAPI(, +# , +# auth_type='oAuth2', +# proxy_url=proxy_url) r = api.request('application/rate_limit_status') - # Print HTTP status code (=200 when no errors). print(r.status_code) @@ -34,4 +28,4 @@ # Parse the JSON response. j = r.response.json() -print(j['resources']['search']) +print(j['resources']['search']) \ No newline at end of file diff --git a/examples/delete_last_tweet.py b/examples/v1.1/delete_last_tweet.py similarity index 88% rename from examples/delete_last_tweet.py rename to examples/v1.1/delete_last_tweet.py index cbf21b9..3707079 100644 --- a/examples/delete_last_tweet.py +++ b/examples/v1.1/delete_last_tweet.py @@ -8,7 +8,6 @@ , ) - class DeleteTweet(Thread): def __init__(self, tweet_id, count): @@ -18,7 +17,7 @@ def __init__(self, tweet_id, count): def run(self): r = api.request('statuses/destroy/:%d' % self.tweet_id) - print(self.count if r.status_code == 200 else r.text) + print(self.count if r.status_code == 200 else 'PROBLEM: ' + r.text) try: count = 0 @@ -32,4 +31,4 @@ def run(self): else: raise Exception(item) except Exception as e: - print('Stopping: %s' % str(e)) + print('Stopping: %s' % str(e)) \ No newline at end of file diff --git a/examples/v1.1/direct_message.py b/examples/v1.1/direct_message.py new file mode 100644 index 0000000..4e234aa --- /dev/null +++ b/examples/v1.1/direct_message.py @@ -0,0 +1,27 @@ +from TwitterAPI import TwitterAPI +import json + +api = TwitterAPI(, + , + , + ) + +user_id = +message_text = + +event = { + "event": { + "type": "message_create", + "message_create": { + "target": { + "recipient_id": user_id + }, + "message_data": { + "text": message_text + } + } + } +} + +r = api.request('direct_messages/events/new', json.dumps(event)) +print('SUCCESS' if r.status_code == 200 else 'PROBLEM: ' + r.text) \ No newline at end of file diff --git a/examples/v1.1/dump_timeline.py b/examples/v1.1/dump_timeline.py new file mode 100644 index 0000000..7f56230 --- /dev/null +++ b/examples/v1.1/dump_timeline.py @@ -0,0 +1,23 @@ +# Print a user's timeline. This will get up to 3,200 tweets, which +# is the maximum the Twitter API allows. + +from TwitterAPI import TwitterAPI, TwitterPager + +SCREEN_NAME = 'TheTweetOfGod' + +api = TwitterAPI(, + , + auth_type='oAuth2') + +pager = TwitterPager(api, + 'statuses/user_timeline', + {'screen_name':SCREEN_NAME, 'count':200}) + +count = 0 +for item in pager.get_iterator(wait=3.5): + if 'text' in item: + count = count + 1 + print(count, item['text']) + elif 'message' in item: + print(item['message']) + break \ No newline at end of file diff --git a/examples/v1.1/logging.py b/examples/v1.1/logging.py new file mode 100644 index 0000000..6bb29b2 --- /dev/null +++ b/examples/v1.1/logging.py @@ -0,0 +1,31 @@ +from TwitterAPI import TwitterAPI +import logging + +# SET UP LOGGING TO FILE AND TO CONSOLE +formatter = logging.Formatter('%(levelname)s %(asctime)s %(message)s', + '%m/%d/%Y %I:%M:%S %p') +fh = logging.FileHandler('logging.log') +fh.setFormatter(formatter) +ch = logging.StreamHandler() +ch.setFormatter(formatter) +logger = logging.getLogger() +logger.setLevel(logging.INFO) +logger.addHandler(fh) +logger.addHandler(ch) + +api = TwitterAPI(, + , + , + ) + +logging.info('START SAMPLE STREAM') + +try: + r = api.request('statuses/sample') + for item in r: + if 'text' in item: + print(item['text']) + elif: + logging.info('NOT A TWEET: %s' % item.text) +except Exception as e: + logging.warning('STOPPING: %s' % str(e)) \ No newline at end of file diff --git a/examples/v1.1/lookup_tweet.py b/examples/v1.1/lookup_tweet.py new file mode 100644 index 0000000..1a8f950 --- /dev/null +++ b/examples/v1.1/lookup_tweet.py @@ -0,0 +1,12 @@ +from TwitterAPI import TwitterAPI + +TWEET_ID = '964575983633252353' + +api = TwitterAPI(, + , + auth_type='oAuth2') + +r = api.request('statuses/show/:' + TWEET_ID) +tweet = r.json() +print(tweet['user']['screen_name'] + ':' + tweet['text'] if r.status_code == 200 + else 'PROBLEM: ' + r.text) \ No newline at end of file diff --git a/examples/v1.1/lookup_user.py b/examples/v1.1/lookup_user.py new file mode 100644 index 0000000..1b1a81a --- /dev/null +++ b/examples/v1.1/lookup_user.py @@ -0,0 +1,10 @@ +from TwitterAPI import TwitterAPI + +SCREEN_NAME = 'TheTweetOfGod' + +api = TwitterAPI(, + , + auth_type='oAuth2') + +r = api.request('users/lookup', {'screen_name':SCREEN_NAME}) +print(r.json()[0]['id'] if r.status_code == 200 else 'PROBLEM: ' + r.text) diff --git a/examples/v1.1/page_tweets.py b/examples/v1.1/page_tweets.py new file mode 100644 index 0000000..6a80931 --- /dev/null +++ b/examples/v1.1/page_tweets.py @@ -0,0 +1,13 @@ +from TwitterAPI import TwitterAPI, TwitterPager + +SEARCH_TERM = 'pizza' + +api = TwitterAPI(, + , + , + ) + +pager = TwitterPager(api, 'search/tweets', {'q': SEARCH_TERM}) + +for item in pager.get_iterator(): + print(item['text'] if 'text' in item else item) \ No newline at end of file diff --git a/examples/v1.1/post_tweet.py b/examples/v1.1/post_tweet.py new file mode 100644 index 0000000..d5977f2 --- /dev/null +++ b/examples/v1.1/post_tweet.py @@ -0,0 +1,11 @@ +from TwitterAPI import TwitterAPI + +TWEET_TEXT = "Ce n'est pas un tweet tweet." + +api = TwitterAPI(, + , + , + ) + +r = api.request('statuses/update', {'status': TWEET_TEXT}) +print('SUCCESS' if r.status_code == 200 else 'PROBLEM: ' + r.text) \ No newline at end of file diff --git a/examples/v1.1/premium_search.py b/examples/v1.1/premium_search.py new file mode 100644 index 0000000..f5426ff --- /dev/null +++ b/examples/v1.1/premium_search.py @@ -0,0 +1,16 @@ +from TwitterAPI import TwitterAPI + +SEARCH_TERM = 'pizza' +PRODUCT = '30day' +LABEL = 'your label' + +api = TwitterAPI(, + , + , + ) + +r = api.request('tweets/search/%s/:%s' % (PRODUCT, LABEL), + {'query':SEARCH_TERM}) + +for item in r: + print(item['text'] if 'text' in item else item) diff --git a/examples/sample_freq.py b/examples/v1.1/sample_frequency.py similarity index 66% rename from examples/sample_freq.py rename to examples/v1.1/sample_frequency.py index 3d6ef31..8615f0d 100644 --- a/examples/sample_freq.py +++ b/examples/v1.1/sample_frequency.py @@ -1,27 +1,10 @@ -from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterConnectionError, TwitterRequestError +from TwitterAPI import TwitterAPI, TwitterConnectionError, TwitterRequestError import time -import logging - - -# SET UP LOGGING TO FILE AND TO CONSOLE -formatter = logging.Formatter('%(levelname)s %(asctime)s %(message)s', - '%m/%d/%Y %I:%M:%S %p') -fh = logging.FileHandler('sample_freq.log') -fh.setFormatter(formatter) -ch = logging.StreamHandler() -ch.setFormatter(formatter) -logger = logging.getLogger() -logger.setLevel(logging.INFO) -logger.addHandler(fh) -logger.addHandler(ch) - - -o = TwitterOAuth.read_file() -api = TwitterAPI(o.consumer_key, - o.consumer_secret, - o.access_token_key, - o.access_token_secret) +api = TwitterAPI(, + , + , + ) class Frequency: @@ -48,9 +31,11 @@ def update(self): self.interval_start = now self.interval_count = 0 - freq = Frequency() +# THIS DEMONSTRATES HOW TO HANDLE ALL TYPES OF STREAMING ERRORS. +# SO, THIS IS APPLICAPLE TO 'statuses/filter' AS WELL. ONLY WHEN +# APPROPRIATE, A DROPPED CONNECTION IS RE-ESTABLISHED. while True: try: @@ -59,15 +44,15 @@ def update(self): if 'text' in item: freq.update() elif 'limit' in item: - logging.info('TWEETS SKIPPED: %s' % item['limit']['track']) + print('TWEETS SKIPPED: %s' % item['limit']['track']) elif 'warning' in item: - logging.warning(item['warning']) + print(item['warning']) elif 'disconnect' in item: event = item['disconnect'] if event['code'] in [2,5,6,7]: # streaming connection rejected raise Exception(event) - logging.info('RE-CONNECTING: %s' % event) + print('RE-CONNECTING: %s' % event) break except TwitterRequestError as e: if e.status_code < 500: @@ -80,4 +65,4 @@ def update(self): break except Exception as e: print('STOPPED: %s %s' % (type(e), e)) - break + break \ No newline at end of file diff --git a/examples/v1.1/search_tweets.py b/examples/v1.1/search_tweets.py new file mode 100644 index 0000000..4067281 --- /dev/null +++ b/examples/v1.1/search_tweets.py @@ -0,0 +1,15 @@ +from TwitterAPI import TwitterAPI + +SEARCH_TERM = 'pizza' + +api = TwitterAPI(, + , + , + ) + +r = api.request('search/tweets', {'q': SEARCH_TERM}) + +for item in r: + print(item['text'] if 'text' in item else item) + +print('\nQUOTA: %s' % r.get_quota()) \ No newline at end of file diff --git a/examples/v1.1/stream_tweets.py b/examples/v1.1/stream_tweets.py new file mode 100644 index 0000000..51da26b --- /dev/null +++ b/examples/v1.1/stream_tweets.py @@ -0,0 +1,13 @@ +from TwitterAPI import TwitterAPI + +TRACK_TERM = 'pizza' + +api = TwitterAPI(, + , + , + ) + +r = api.request('statuses/filter', {'track': TRACK_TERM}) + +for item in r: + print(item['text'] if 'text' in item else item) \ No newline at end of file diff --git a/examples/unfollow_friends.py b/examples/v1.1/unfollow_friends.py similarity index 52% rename from examples/unfollow_friends.py rename to examples/v1.1/unfollow_friends.py index 8c2838e..65a0cb5 100644 --- a/examples/unfollow_friends.py +++ b/examples/v1.1/unfollow_friends.py @@ -1,27 +1,15 @@ -''' -Unfollow friends that do not follow you. -''' - +# Unfollow friends that do not follow you. from TwitterAPI import TwitterAPI +api = TwitterAPI(, + , + , + ) -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN = '' -ACCESS_TOKEN_SECRET = '' - - -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN, - ACCESS_TOKEN_SECRET) - - -followers = set(id for id in api.request('followers/ids')) friends = set(id for id in api.request('friends/ids')) -unfollow = set(followers) - set(friends) - +followers = set(id for id in api.request('followers/ids')) +unfollow = set(friends) - set(followers) for id in unfollow: r = api.request('friendships/destroy', {'user_id': id}) diff --git a/examples/v1.1/upload_image.py b/examples/v1.1/upload_image.py new file mode 100644 index 0000000..11aa5a2 --- /dev/null +++ b/examples/v1.1/upload_image.py @@ -0,0 +1,23 @@ +# Post an image with tweet. Requires two API calls. + +from TwitterAPI import TwitterAPI + +TWEET_TEXT = 'some tweet text' +IMAGE_PATH = './some_image.png' + +api = TwitterAPI(, + , + , + ) + +# STEP 1 - upload image +file = open(IMAGE_PATH, 'rb') +data = file.read() +r = api.request('media/upload', None, {'media': data}) +print('UPLOAD MEDIA SUCCESS' if r.status_code == 200 else 'UPLOAD MEDIA FAILURE: ' + r.text) + +# STEP 2 - post tweet with a reference to uploaded image +if r.status_code == 200: + media_id = r.json()['media_id'] + r = api.request('statuses/update', {'status': TWEET_TEXT, 'media_ids': media_id}) + print('UPDATE STATUS SUCCESS' if r.status_code == 200 else 'UPDATE STATUS FAILURE: ' + r.text) \ No newline at end of file diff --git a/examples/v1.1/upload_tweet.py b/examples/v1.1/upload_tweet.py new file mode 100644 index 0000000..f9e7b45 --- /dev/null +++ b/examples/v1.1/upload_tweet.py @@ -0,0 +1,11 @@ +from TwitterAPI import TwitterAPI + +TWEET_TEXT = 'some tweet text' + +api = TwitterAPI(, + , + , + ) + +r = api.request('statuses/update', {'status':TWEET_TEXT}) +print('SUCCESS' if r.status_code == 200 else 'PROBLEM: ' + r.text) \ No newline at end of file diff --git a/examples/upload_video.py b/examples/v1.1/upload_video.py similarity index 64% rename from examples/upload_video.py rename to examples/v1.1/upload_video.py index 41356bf..dc29c94 100644 --- a/examples/upload_video.py +++ b/examples/v1.1/upload_video.py @@ -8,34 +8,25 @@ import os import sys - VIDEO_FILENAME = 'test.mp4' TWEET_TEXT = 'Video upload test' -CONSUMER_KEY = '' -CONSUMER_SECRET = '' -ACCESS_TOKEN_KEY = '' -ACCESS_TOKEN_SECRET = '' - - -api = TwitterAPI(CONSUMER_KEY, - CONSUMER_SECRET, - ACCESS_TOKEN_KEY, - ACCESS_TOKEN_SECRET) - +api = TwitterAPI(, + , + , + ) bytes_sent = 0 total_bytes = os.path.getsize(VIDEO_FILENAME) file = open(VIDEO_FILENAME, 'rb') - def check_status(r): + # EXIT PROGRAM WITH ERROR MESSAGE if r.status_code < 200 or r.status_code > 299: print(r.status_code) print(r.text) sys.exit(0) - r = api.request('media/upload', {'command':'INIT', 'media_type':'video/mp4', 'total_bytes':total_bytes}) check_status(r) @@ -43,15 +34,15 @@ def check_status(r): segment_id = 0 while bytes_sent < total_bytes: - chunk = file.read(4*1024*1024) - r = api.request('media/upload', {'command':'APPEND', 'media_id':media_id, 'segment_index':segment_id}, {'media':chunk}) - check_status(r) - segment_id = segment_id + 1 - bytes_sent = file.tell() - print('[' + str(total_bytes) + ']', str(bytes_sent)) + chunk = file.read(4*1024*1024) + r = api.request('media/upload', {'command':'APPEND', 'media_id':media_id, 'segment_index':segment_id}, {'media':chunk}) + check_status(r) + segment_id = segment_id + 1 + bytes_sent = file.tell() + print('[' + str(total_bytes) + ']', str(bytes_sent)) r = api.request('media/upload', {'command':'FINALIZE', 'media_id':media_id}) check_status(r) r = api.request('statuses/update', {'status':TWEET_TEXT, 'media_ids':media_id}) -check_status(r) +check_status(r) \ No newline at end of file diff --git a/examples/v2/conversation_tree.py b/examples/v2/conversation_tree.py new file mode 100644 index 0000000..9ec6b54 --- /dev/null +++ b/examples/v2/conversation_tree.py @@ -0,0 +1,122 @@ +from TwitterAPI import (TwitterAPI, TwitterOAuth, TwitterRequestError, + TwitterConnectionError, TwitterPager, HydrateType) + + +# NOTE: If any of the conversation is over a week old then it will not get +# returned unless you are using academic credentials. +CONVERSATION_ID = '20' + + +# +# UTILITY CLASS +# + +class TreeNode: + """TreeNode is used to organize tweets as a tree structure""" + + def __init__(self, data): + """data is a tweet's json object""" + self.data = data + self.children = [] + self.replied_to_tweet = None + if 'referenced_tweets' in self.data: + for tweet in self.data['referenced_tweets']: + if tweet['type'] == 'replied_to': + self.replied_to_tweet = tweet['id'] + break + + def id(self): + """a node is identified by its tweet id""" + return self.data['id'] + + def parent(self): + """the reply-to tweet is the parent of the node""" + return self.replied_to_tweet + + def find_parent_of(self, node): + """append a node to the children of it's parent tweet""" + if node.parent() == self.id(): + self.children.append(node) + return True + for child in self.children: + if child.find_parent_of(node): + return True + return False + + def print_tree(self, level): + """level 0 is the root node, then incremented for subsequent generations""" + created_at = self.data['created_at'] + username = self.data['author_id']['username'] + text_80chars = self.data['text'][0:80].replace('\n', ' ') + print(f'{level*"_"}{level}: [{created_at}][{username}] {text_80chars}') + level += 1 + for child in reversed(self.children): + child.print_tree(level) + + +# +# PROGRAM BEGINS HERE +# + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + + # GET ROOT OF THE CONVERSATION + + r = api.request(f'tweets/:{CONVERSATION_ID}', + { + 'expansions':'author_id', + 'tweet.fields':'author_id,conversation_id,created_at,referenced_tweets' + }, + hydrate_type=HydrateType.REPLACE) + + root = None + for item in r: + root = TreeNode(item) + print(f'ROOT {root.id()}') + if not root: + print(f'Conversation ID {CONVERSATION_ID} does not exist') + exit() + + # GET ALL REPLIES IN CONVERSATION + # (RETURNED IN REVERSE CHRONOLOGICAL ORDER) + + pager = TwitterPager(api, 'tweets/search/recent', + { + 'query':f'conversation_id:{CONVERSATION_ID}', + 'expansions':'author_id', + 'tweet.fields':'author_id,conversation_id,created_at,referenced_tweets' + }, + hydrate_type=HydrateType.REPLACE) + + # "wait=2" means wait 2 seconds between each request. + # The rate limit is 450 requests per 15 minutes, or 1 request every 15*60/450 = 2 seconds. + + orphans = [] + + for item in pager.get_iterator(wait=2): + node = TreeNode(item) + print(f'{node.id()} => {node.parent()}', item['author_id']['username']) + # COLLECT ANY ORPHANS THAT ARE CHILDREN OF THE NEW NODE + orphans = [orphan for orphan in orphans if not node.find_parent_of(orphan)] + # IF THE NEW NODE CANNOT BE PLACED IN TREE, ORPHAN IT UNTIL ITS PARENT IS FOUND + if not root.find_parent_of(node): + orphans.append(node) + + print('\nTREE...') + root.print_tree(0) + + # YOU MIGHT GET ORPHANS WHEN PART OF THE CONVERSATION IS OLDER THAN A WEEK + assert len(orphans) == 0, f'{len(orphans)} orphaned tweets' + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/expansions_media.py b/examples/v2/expansions_media.py new file mode 100644 index 0000000..2db13d2 --- /dev/null +++ b/examples/v2/expansions_media.py @@ -0,0 +1,31 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +TWEET_ID = '1318585013944942593' +EXPANSIONS = 'attachments.media_keys' +MEDIA_FIELDS = 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets/:{TWEET_ID}', + { + 'expansions':EXPANSIONS, + 'media.fields':MEDIA_FIELDS + }) + + for item in r: + print(json.dumps(item, indent=2)) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2/expansions_place.py b/examples/v2/expansions_place.py new file mode 100644 index 0000000..671bd49 --- /dev/null +++ b/examples/v2/expansions_place.py @@ -0,0 +1,31 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +TWEET_ID = '1295386980495364101' +EXPANSIONS = 'geo.place_id' +PLACE_FIELDS = 'contained_within,country,country_code,full_name,geo,id,name,place_type' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets/:{TWEET_ID}', + { + 'expansions':EXPANSIONS, + 'place.fields':PLACE_FIELDS + }) + + for item in r: + print(json.dumps(item, indent=2)) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2/expansions_poll.py b/examples/v2/expansions_poll.py new file mode 100644 index 0000000..72b9754 --- /dev/null +++ b/examples/v2/expansions_poll.py @@ -0,0 +1,31 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +TWEET_ID = '1317617296978833408' +EXPANSIONS = 'attachments.poll_ids' +POLL_FIELDS = 'duration_minutes,end_datetime,id,options,voting_status' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets/:{TWEET_ID}', + { + 'expansions':EXPANSIONS, + 'poll.fields':POLL_FIELDS + }) + + for item in r: + print(json.dumps(item, indent=2)) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2/expansions_tweet.py b/examples/v2/expansions_tweet.py new file mode 100644 index 0000000..0f125a4 --- /dev/null +++ b/examples/v2/expansions_tweet.py @@ -0,0 +1,31 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +TWEET_ID = '1275914014867050499' +EXPANSIONS = 'referenced_tweets.id' +TWEET_FIELDS = 'attachments,author_id,context_annotations,conversation_id,created_at,entities,geo,id,in_reply_to_user_id,lang,public_metrics,text' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets/:{TWEET_ID}', + { + 'expansions':EXPANSIONS, + 'tweet.fields':TWEET_FIELDS + }) + + for item in r: + print(json.dumps(item, indent=2)) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2/expansions_user.py b/examples/v2/expansions_user.py new file mode 100644 index 0000000..70f43ce --- /dev/null +++ b/examples/v2/expansions_user.py @@ -0,0 +1,31 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +TWEET_ID = '1275914014867050499' +EXPANSIONS = 'author_id,in_reply_to_user_id,referenced_tweets.id.author_id' +USER_FIELDS = 'location,profile_image_url,verified' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets/:{TWEET_ID}', + { + 'expansions':EXPANSIONS, + 'user.fields':USER_FIELDS + }) + + for item in r: + print(json.dumps(item, indent=2)) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2/lookup_tweet.py b/examples/v2/lookup_tweet.py new file mode 100644 index 0000000..c1e2dc6 --- /dev/null +++ b/examples/v2/lookup_tweet.py @@ -0,0 +1,24 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError + +TWEET_ID = '964575983633252353' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets/:{TWEET_ID}') + + for item in r: + print(item) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2/lookup_tweet_hydrate.py b/examples/v2/lookup_tweet_hydrate.py new file mode 100644 index 0000000..00bde04 --- /dev/null +++ b/examples/v2/lookup_tweet_hydrate.py @@ -0,0 +1,35 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, HydrateType +import json + +TWEET_ID = '1360029234773770240' + +EXPANSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,in_reply_to_user_id,attachments.media_keys,attachments.poll_ids,geo.place_id,entities.mentions.username' +MEDIA_FIELDS = 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics' +TWEET_FIELDS = 'created_at,author_id,public_metrics,context_annotations,entities' +USER_FIELDS = 'location,profile_image_url,verified' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets/:{TWEET_ID}', + { + 'expansions': EXPANSIONS, + 'tweet.fields': TWEET_FIELDS, + 'user.fields': USER_FIELDS, + 'media.fields': MEDIA_FIELDS, + }, + hydrate_type=HydrateType.APPEND) + + for item in r: + print(json.dumps(item, indent=2, sort_keys=True)) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/lookup_tweets.py b/examples/v2/lookup_tweets.py new file mode 100644 index 0000000..aa1e370 --- /dev/null +++ b/examples/v2/lookup_tweets.py @@ -0,0 +1,24 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError + +TWEET_IDS = ['964575983633252353', '1497014169178185730'] + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'tweets',{'ids':",".join(TWEET_IDS)}) + + for item in r: + print(item) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2/lookup_user.py b/examples/v2/lookup_user.py new file mode 100644 index 0000000..752215a --- /dev/null +++ b/examples/v2/lookup_user.py @@ -0,0 +1,24 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError + +SCREEN_NAME = 'TheTweetOfGod' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request(f'users/by/username/:{SCREEN_NAME}') + + for item in r: + print(item) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/page_tweets.py b/examples/v2/page_tweets.py new file mode 100644 index 0000000..1dfb618 --- /dev/null +++ b/examples/v2/page_tweets.py @@ -0,0 +1,22 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager + +QUERY = 'pizza' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + pager = TwitterPager(api, 'tweets/search/recent', {'query':QUERY}) + + for item in pager.get_iterator(new_tweets=False): + print(item) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/page_tweets_hydrate.py b/examples/v2/page_tweets_hydrate.py new file mode 100644 index 0000000..e32117c --- /dev/null +++ b/examples/v2/page_tweets_hydrate.py @@ -0,0 +1,36 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, TwitterPager, HydrateType +import json + +QUERY = 'pizza' + +EXPANSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,in_reply_to_user_id,attachments.media_keys,attachments.poll_ids,geo.place_id,entities.mentions.username' +MEDIA_FIELDS = 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics' +TWEET_FIELDS = 'created_at,author_id,public_metrics' +USER_FIELDS = 'location,profile_image_url,verified' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + pager = TwitterPager(api, 'tweets/search/recent', + { + 'query': {QUERY}, + 'expansions': EXPANSIONS, + 'tweet.fields': TWEET_FIELDS, + 'user.fields': USER_FIELDS, + 'media.fields': MEDIA_FIELDS, + }, + hydrate_type=HydrateType.APPEND) + + for item in pager.get_iterator(new_tweets=False): + print(json.dumps(item, indent=2)) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/rules_add.py b/examples/v2/rules_add.py new file mode 100644 index 0000000..733133e --- /dev/null +++ b/examples/v2/rules_add.py @@ -0,0 +1,24 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +QUERY = 'pizza"' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + + # ADD STREAM RULES + + r = api.request('tweets/search/stream/rules', {'add': [{'value':QUERY}]}) + print(f'[{r.status_code}] RULE ADDED: {json.dumps(r.json(), indent=2)}\n') + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/rules_del.py b/examples/v2/rules_del.py new file mode 100644 index 0000000..752cc93 --- /dev/null +++ b/examples/v2/rules_del.py @@ -0,0 +1,27 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +RULE_IDS = [ + "1317935598431309824", + "1321813278532751361", +] + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + + # DELETE STREAM RULES + + r = api.request('tweets/search/stream/rules', {'delete': {'ids':RULE_IDS}}) + print(f'[{r.status_code}] RULES DELETED: {json.dumps(r.json(), indent=2)}\n') + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/rules_del_all.py b/examples/v2/rules_del_all.py new file mode 100644 index 0000000..5e64748 --- /dev/null +++ b/examples/v2/rules_del_all.py @@ -0,0 +1,33 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + + # GET STREAM RULES + + rule_ids = [] + r = api.request('tweets/search/stream/rules', method_override='GET') + for item in r: + if 'id' in item: + rule_ids.append(item['id']) + else: + print(json.dumps(item, indent=2)) + + # DELETE STREAM RULES + + if len(rule_ids) > 0: + r = api.request('tweets/search/stream/rules', {'delete': {'ids':rule_ids}}) + print(f'[{r.status_code}] RULES DELETED: {json.dumps(r.json(), indent=2)}\n') + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/rules_get.py b/examples/v2/rules_get.py new file mode 100644 index 0000000..1a0c80d --- /dev/null +++ b/examples/v2/rules_get.py @@ -0,0 +1,22 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +import json + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + + # GET STREAM RULES + + r = api.request('tweets/search/stream/rules', method_override='GET') + print(f'[{r.status_code}] RULES: {json.dumps(r.json(), indent=2)}\n') + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/search_tweets.py b/examples/v2/search_tweets.py new file mode 100644 index 0000000..9928615 --- /dev/null +++ b/examples/v2/search_tweets.py @@ -0,0 +1,31 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError + +QUERY = 'pizza' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request('tweets/search/recent', { + 'query':QUERY, + 'tweet.fields':'author_id', + 'expansions':'author_id'}) + + for item in r: + print(item) + + print('\nINCLUDES') + print(r.json()['includes']) + + print('\nQUOTA') + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2/search_tweets_hydrate.py b/examples/v2/search_tweets_hydrate.py new file mode 100644 index 0000000..0704a99 --- /dev/null +++ b/examples/v2/search_tweets_hydrate.py @@ -0,0 +1,37 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, HydrateType +import json + +QUERY = 'kitten' +EXPANSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,in_reply_to_user_id,attachments.media_keys' +MEDIA_FIELDS = 'duration_ms,height,media_key,preview_image_url,type,url,width,public_metrics' +TWEET_FIELDS = 'created_at,author_id,public_metrics' +USER_FIELDS = 'location,profile_image_url,verified' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + r = api.request('tweets/search/recent', + { + 'query': {QUERY}, + 'expansions': EXPANSIONS, + 'media.fields': MEDIA_FIELDS, + 'tweet.fields': TWEET_FIELDS, + 'user.fields': USER_FIELDS, + }, + hydrate_type=HydrateType.APPEND) + + for item in r: + print(json.dumps(item, indent=2)) + + print(r.get_quota()) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/stream_forever.py b/examples/v2/stream_forever.py new file mode 100644 index 0000000..67e19d7 --- /dev/null +++ b/examples/v2/stream_forever.py @@ -0,0 +1,120 @@ +from TwitterAPI import (TwitterAPI, + TwitterOAuth, + TwitterRequestError, + TwitterConnectionError, + HydrateType) + + +RULES = ['pizza','lambrusco'] + +EXPANSIONS = 'author_id' +USER_FIELDS = 'created_at,description,location,name,username' +TWEET_FIELDS = 'author_id,created_at,entities,id,lang,public_metrics,source,text' + + +## +## SET UP LOGGING TO FILE AND TO CONSOLE +## + +import logging +formatter = logging.Formatter('%(levelname)s %(asctime)s %(message)s', '%m/%d/%Y %I:%M:%S %p') +fh = logging.FileHandler('stream_forever.log') +sh = logging.StreamHandler() +fh.setFormatter(formatter) +sh.setFormatter(formatter) +logger = logging.getLogger() +logger.setLevel(logging.INFO) +logger.addHandler(fh) +logger.addHandler(sh) + + +## +## AUTHENTICATE WITH TWITTER +## + +o = TwitterOAuth.read_file() +api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + + +## +## STREAMING RULES +## + +# ADD RULES (NO HARM RE-ADDING, BUT ONCE IS ENOUGH) +for rule in RULES: + r = api.request('tweets/search/stream/rules', {'add': [{'value':rule}]}) + print(f'[{r.status_code}] RULE ADDED: {r.text}') + if r.status_code != 201: exit() + +# PRINT RULES +r = api.request('tweets/search/stream/rules', method_override='GET') +print(f'\n[{r.status_code}] RULES:') +if r.status_code != 200: exit() +for item in r: + print(item['value']) + + +## +## "BACK OFF" STRATEGY FOR DISCONNECTS AND RATE LIMITS +## USE 60 SECOND INTERVAL FOR HTTP 429 ERRORS +## USE 5 SECOND INTERVAL FOR ALL OTHER HTTP ERRORS +## REFER TO: https://developer.twitter.com/en/docs/twitter-api/tweets/filtered-stream/integrate/handling-disconnections +## + +from time import sleep + +backOffCount = 0 + +def backOff(interval): + global backOffCount, logger + seconds = pow(2, backOffCount) * interval + backOffCount = min(backOffCount + 1, 6) + logger.info(f'Back off {seconds} seconds') + sleep(seconds) + + +## +## STREAM FOREVER... +## + +while True: + try: + # START STREAM + logger.info('START STREAM...') + r = api.request('tweets/search/stream', + { + 'expansions': EXPANSIONS, + 'user.fields': USER_FIELDS, + 'tweet.fields': TWEET_FIELDS + }, + hydrate_type=HydrateType.APPEND) + + if r.status_code == 200: + logger.info(f'[{r.status_code}] STREAM CONNECTED') + backOffCount = 0 + else: + logger.info(f'[{r.status_code}] FAILED TO CONNECT. REASON:\n{r.text}') + backOff(60 if r.status_code == 429 else 5) + continue + + for item in r: + if 'data' in item: + print(item['data']['text']) + elif 'errors' in item: + logger.error(item['errors']) + else: + # PROBABLY SHOULD NEVER BRANCH TO HERE + logger.warning(f'UNKNOWN ITEM TYPE: {item}') + + except TwitterConnectionError: + backOff(5) + continue + + except TwitterRequestError: + break + + except KeyboardInterrupt: + break + + finally: + logger.info('STREAM STOPPED') \ No newline at end of file diff --git a/examples/v2/stream_tweets.py b/examples/v2/stream_tweets.py new file mode 100644 index 0000000..222a1ea --- /dev/null +++ b/examples/v2/stream_tweets.py @@ -0,0 +1,38 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError + +QUERY = 'pizza' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type='oAuth2', api_version='2') + + # ADD STREAM RULES + + r = api.request('tweets/search/stream/rules', {'add': [{'value':QUERY}]}) + print(f'[{r.status_code}] RULE ADDED: {r.text}') + if r.status_code != 201: exit() + + # GET STREAM RULES + + r = api.request('tweets/search/stream/rules', method_override='GET') + print(f'[{r.status_code}] RULES: {r.text}') + if r.status_code != 200: exit() + + # START STREAM + + r = api.request('tweets/search/stream') + print(f'[{r.status_code}] START...') + if r.status_code != 200: exit() + for item in r: + print(item) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) \ No newline at end of file diff --git a/examples/v2/stream_tweets_hydrate.py b/examples/v2/stream_tweets_hydrate.py new file mode 100644 index 0000000..4fb93d5 --- /dev/null +++ b/examples/v2/stream_tweets_hydrate.py @@ -0,0 +1,51 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError, HydrateType, OAuthType +import json + +QUERY = '"pizza" OR "burger"' +EXPANSIONS = 'author_id,referenced_tweets.id,referenced_tweets.id.author_id,in_reply_to_user_id,attachments.media_keys,attachments.poll_ids,geo.place_id,entities.mentions.username' +TWEET_FIELDS='author_id,conversation_id,created_at,entities,geo,id,lang,public_metrics,source,text' +USER_FIELDS='created_at,description,entities,location,name,profile_image_url,public_metrics,url,username' + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, auth_type=OAuthType.OAUTH2, api_version='2') + + # ADD STREAM RULES + + r = api.request('tweets/search/stream/rules', {'add': [{'value':QUERY}]}) + print(f'[{r.status_code}] RULE ADDED: {json.dumps(r.json(), indent=2)}\n') + if r.status_code != 201: exit() + + # GET STREAM RULES + + r = api.request('tweets/search/stream/rules', method_override='GET') + print(f'[{r.status_code}] RULES: {json.dumps(r.json(), indent=2)}\n') + if r.status_code != 200: exit() + + # START STREAM + + r = api.request('tweets/search/stream', { + 'expansions': EXPANSIONS, + 'tweet.fields': TWEET_FIELDS, + 'user.fields': USER_FIELDS, + }, + hydrate_type=HydrateType.APPEND) + + print(f'[{r.status_code}] START...') + if r.status_code != 200: exit() + for item in r: + print(json.dumps(item, indent=2)) + +except KeyboardInterrupt: + print('\nDone!') + +except TwitterRequestError as e: + print(f'\n{e.status_code}') + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/user_bookmarks.py b/examples/v2/user_bookmarks.py new file mode 100644 index 0000000..87cbbe1 --- /dev/null +++ b/examples/v2/user_bookmarks.py @@ -0,0 +1,44 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +from pprint import pprint + +USER_ID = '1249069835562233858' # https://twitter.com/bascodes +access_token = "" + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(oauth2_access_token=access_token, api_version='2', auth_type="oAuth2User") + + # Get tweets - default setting + tweets = api.request(f'users/:{USER_ID}/bookmarks') + for t in tweets: + print(t) + + # Get tweets with customization - (5 tweets only with created_at timestamp) + print() + params = {'max_results': 5, 'tweet.fields': 'created_at'} + tweets = api.request(f'users/:{USER_ID}/bookmarks', params) + for t in tweets: + pprint(t) + + # Get next 5 tweets + print() + next_token = tweets.json()['meta']['next_token'] + params = {'max_results': 5, 'tweet.fields': 'created_at', 'pagination_token': next_token} + tweets = api.request(f'users/:{USER_ID}/bookmarks', params) + for t in tweets: + pprint(t) + + +except TwitterRequestError as e: + print('Request error') + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print('Connection error') + print(e) + +except Exception as e: + print('Exception') + print(e) \ No newline at end of file diff --git a/examples/v2/user_followers_following.py b/examples/v2/user_followers_following.py new file mode 100644 index 0000000..e3a0db8 --- /dev/null +++ b/examples/v2/user_followers_following.py @@ -0,0 +1,28 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError + +USER_ID = '2244994945' # https://twitter.com/TwitterDev + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + + # Get followers + followers = api.request(f'users/:{USER_ID}/followers') + for f in followers: + print(f) + + # Get following + following = api.request(f'users/:{USER_ID}/following') + for f in following: + print(f) + +except TwitterRequestError as e: + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print(e) + +except Exception as e: + print(e) diff --git a/examples/v2/user_tweets.py b/examples/v2/user_tweets.py new file mode 100644 index 0000000..63ce81f --- /dev/null +++ b/examples/v2/user_tweets.py @@ -0,0 +1,43 @@ +from TwitterAPI import TwitterAPI, TwitterOAuth, TwitterRequestError, TwitterConnectionError +from pprint import pprint + +USER_ID = '2244994945' # https://twitter.com/TwitterDev + +try: + o = TwitterOAuth.read_file() + api = TwitterAPI(o.consumer_key, o.consumer_secret, o.access_token_key, o.access_token_secret, api_version='2') + + # Get tweets - default setting + tweets = api.request(f'users/:{USER_ID}/tweets') + for t in tweets: + print(t) + + # Get tweets with customization - (5 tweets only with created_at timestamp) + print() + params = {'max_results': 5, 'tweet.fields': 'created_at'} + tweets = api.request(f'users/:{USER_ID}/tweets', params) + for t in tweets: + pprint(t) + + # Get next 5 tweets + print() + next_token = tweets.json()['meta']['next_token'] + params = {'max_results': 5, 'tweet.fields': 'created_at', 'pagination_token': next_token} + tweets = api.request(f'users/:{USER_ID}/tweets', params) + for t in tweets: + pprint(t) + + +except TwitterRequestError as e: + print('Request error') + print(e.status_code) + for msg in iter(e): + print(msg) + +except TwitterConnectionError as e: + print('Connection error') + print(e) + +except Exception as e: + print('Exception') + print(e) \ No newline at end of file diff --git a/tests/test_iterators.py b/tests/test_iterators.py index ed719a0..b85fe6c 100644 --- a/tests/test_iterators.py +++ b/tests/test_iterators.py @@ -35,10 +35,10 @@ def test_streaming_iterator(self): self.use_iterator(it) def test_paging_iterator(self): - pager = TwitterAPI.TwitterRestPager(self.api, - 'search/tweets', - {'q': 'pizza'}) - self.assertIsInstance(pager, TwitterAPI.TwitterRestPager) + pager = TwitterAPI.TwitterPager(self.api, + 'search/tweets', + {'q': 'pizza'}) + self.assertIsInstance(pager, TwitterAPI.TwitterPager) it = pager.get_iterator() self.use_iterator(it)