diff --git a/.travis.yml b/.travis.yml index 03b5955c7e..543547c164 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,6 @@ language: python +addons: + postgresql: 9.3 before_install: - node --version install: make env node_modules diff --git a/branch.sql b/branch.sql new file mode 100644 index 0000000000..1e373cfe8e --- /dev/null +++ b/branch.sql @@ -0,0 +1,142 @@ +------------------------------------------------------------------------------- +-- https://github.com/gittip/www.gittip.com/pull/1369 + +BEGIN; + + +-- Add new columns + + -- Note: using "user_name" instead of "username" avoids having the same + -- column name in the participants and elsewhere tables. + ALTER TABLE elsewhere ADD COLUMN user_name text; + ALTER TABLE elsewhere ADD COLUMN display_name text; + ALTER TABLE elsewhere ADD COLUMN email text; + ALTER TABLE elsewhere ADD COLUMN avatar_url text; + ALTER TABLE participants ADD COLUMN avatar_url text; + ALTER TABLE elsewhere ADD COLUMN is_team boolean NOT NULL DEFAULT FALSE; + + + +-- Extract info + + -- Extract user_name from user_info + UPDATE elsewhere SET user_name = user_id WHERE platform = 'bitbucket'; + UPDATE elsewhere SET user_name = user_info->'display_name' WHERE platform = 'bountysource'; + UPDATE elsewhere SET user_name = user_info->'login' WHERE platform = 'github'; + UPDATE elsewhere SET user_name = user_info->'username' WHERE platform = 'openstreetmap'; + UPDATE elsewhere SET user_name = user_info->'screen_name' WHERE platform = 'twitter'; + UPDATE elsewhere SET user_name = user_info->'username' WHERE platform = 'venmo'; + + -- Extract display_name from user_info + UPDATE elsewhere SET display_name = user_info->'display_name' WHERE platform = 'bitbucket'; + UPDATE elsewhere SET display_name = user_info->'name' WHERE platform = 'github'; + UPDATE elsewhere SET display_name = user_info->'username' WHERE platform = 'openstreetmap'; + UPDATE elsewhere SET display_name = user_info->'name' WHERE platform = 'twitter'; + UPDATE elsewhere SET display_name = user_info->'display_name' WHERE platform = 'venmo'; + UPDATE elsewhere SET display_name = NULL WHERE display_name = 'None'; + + -- Extract available email addresses + UPDATE elsewhere SET email = user_info->'email' WHERE user_info->'email' LIKE '%@%'; + + -- Extract available avatar URLs + UPDATE elsewhere SET avatar_url = concat('https://www.gravatar.com/avatar/', + user_info->'gravatar_id') + WHERE platform = 'github' + AND user_info->'gravatar_id' != '' + AND user_info->'gravatar_id' != 'None'; + UPDATE elsewhere SET avatar_url = concat('https://www.gravatar.com/avatar/', + md5(lower(trim(email)))) + WHERE email IS NOT NULL AND avatar_url IS NULL; + UPDATE elsewhere SET avatar_url = user_info->'avatar' WHERE platform = 'bitbucket'; + UPDATE elsewhere SET avatar_url = user_info->'avatar_url' + WHERE platform = 'bitbucket' AND avatar_url IS NULL; + UPDATE elsewhere SET avatar_url = substring(user_info->'links', $$u'avatar': {u'href': u'([^']+)$$) + WHERE platform = 'bitbucket' AND avatar_url IS NULL; + UPDATE elsewhere SET avatar_url = user_info->'image_url' WHERE platform = 'bountysource'; + UPDATE elsewhere SET avatar_url = user_info->'avatar_url' WHERE platform = 'github' AND avatar_url IS NULL; + UPDATE elsewhere SET avatar_url = user_info->'img_src' WHERE platform = 'openstreetmap'; + UPDATE elsewhere SET avatar_url = replace(user_info->'profile_image_url_https', '_normal.', '.') + WHERE platform = 'twitter'; + UPDATE elsewhere SET avatar_url = user_info->'profile_picture_url' WHERE platform = 'venmo'; + UPDATE elsewhere SET avatar_url = NULL WHERE avatar_url = 'None'; + -- Propagate avatar_url to participants + UPDATE participants p + SET avatar_url = ( + SELECT avatar_url + FROM elsewhere + WHERE participant = p.username + ORDER BY platform = 'github' DESC, + avatar_url LIKE '%gravatar.com%' DESC + LIMIT 1 + ); + + -- Extract is_team from user_info + UPDATE elsewhere SET is_team = true WHERE platform = 'bitbucket' AND user_info->'is_team' = 'True'; + UPDATE elsewhere SET is_team = true WHERE platform = 'github' AND lower(user_info->'type') = 'organization'; + + + +-- Drop old columns and add new ones + + -- Update user_name constraints + ALTER TABLE elsewhere ALTER COLUMN user_name SET NOT NULL, + ALTER COLUMN user_name DROP DEFAULT; + + -- Replace user_info by a new column of type json (instead of hstore) + ALTER TABLE elsewhere DROP COLUMN user_info, + ADD COLUMN extra_info json; + DROP EXTENSION hstore; + + -- Simplify homepage_top_* tables + ALTER TABLE homepage_top_givers DROP COLUMN gravatar_id, + DROP COLUMN twitter_pic, + ADD COLUMN avatar_url text; + ALTER TABLE homepage_top_receivers DROP COLUMN claimed_time, + DROP COLUMN gravatar_id, + DROP COLUMN twitter_pic, + ADD COLUMN avatar_url text; + + -- The following lets us cast queries to elsewhere_with_participant to get the + -- participant data dereferenced and returned in a composite type along with + -- the elsewhere data. Then we can register orm.Models in the application for + -- both participant and elsewhere_with_participant, and when we cast queries + -- elsewhere.*::elsewhere_with_participant, we'll get a hydrated Participant + -- object at .participant. Woo-hoo! + + CREATE TYPE elsewhere_with_participant AS + ( id integer + , platform text + , user_id text + , user_name text + , display_name text + , email text + , avatar_url text + , extra_info json + , is_locked boolean + , is_team boolean + , participant participants + ); -- If Postgres had type inheritance this would be even awesomer. + + CREATE OR REPLACE FUNCTION load_participant_for_elsewhere (elsewhere) + RETURNS elsewhere_with_participant + AS $$ + SELECT $1.id + , $1.platform + , $1.user_id + , $1.user_name + , $1.display_name + , $1.email + , $1.avatar_url + , $1.extra_info + , $1.is_locked + , $1.is_team + , participants.*::participants + FROM participants + WHERE participants.username = $1.participant + ; + $$ LANGUAGE SQL; + + CREATE CAST (elsewhere AS elsewhere_with_participant) + WITH FUNCTION load_participant_for_elsewhere(elsewhere); + +END; diff --git a/configure-aspen.py b/configure-aspen.py index f9f8f7f299..a31bd7f967 100644 --- a/configure-aspen.py +++ b/configure-aspen.py @@ -1,6 +1,5 @@ from __future__ import division -from importlib import import_module import os import sys import threading @@ -12,7 +11,6 @@ from gittip import canonize, configure_payments from gittip.security import authentication, csrf, x_frame_options from gittip.utils import cache_static, timer -from gittip.elsewhere import platform_classes from aspen import log_dammit @@ -46,12 +44,6 @@ gittip.wireup.envvars(website) tell_sentry = gittip.wireup.make_sentry_teller(website) -# this serves two purposes: -# 1) ensure all platform classes are created (and thus added to platform_classes) -# 2) keep the platform modules around to be added to the context below -platform_modules = {platform: import_module("gittip.elsewhere.%s" % platform) - for platform in platform_classes} - # The homepage wants expensive queries. Let's periodically select into an # intermediate table. @@ -124,7 +116,6 @@ def log_busy_threads(): def add_stuff_to_context(request): request.context['username'] = None - request.context.update(platform_modules) def scab_body_onto_response(response): diff --git a/defaults.env b/defaults.env index 962e2188a5..9656bfebb3 100644 --- a/defaults.env +++ b/defaults.env @@ -16,6 +16,10 @@ STRIPE_PUBLISHABLE_API_KEY=1 BALANCED_API_SECRET=90bb3648ca0a11e1a977026ba7e239a9 +DEBUG=1 # Used by oauthlib to bypass security checks. We need it because when + # running locally the OAuth callbacks are http:, not https:. Of course + # DEBUG shouldn't be set in production. + GITHUB_CLIENT_ID=3785a9ac30df99feeef5 GITHUB_CLIENT_SECRET=e69825fafa163a0b0b6d2424c107a49333d46985 GITHUB_CALLBACK=http://127.0.0.1:8537/on/github/associate @@ -26,23 +30,22 @@ BITBUCKET_CALLBACK=http://127.0.0.1:8537/on/bitbucket/associate TWITTER_CONSUMER_KEY=QBB9vEhxO4DFiieRF68zTA TWITTER_CONSUMER_SECRET=mUymh1hVMiQdMQbduQFYRi79EYYVeOZGrhj27H59H78 -TWITTER_ACCESS_TOKEN=34175404-G6W8Hh19GWuUhIMEXK0LyZsy7N9aCMcy1bYJ9rI -TWITTER_ACCESS_TOKEN_SECRET=K6wxV1OCsihZAkEPkWtoLYDiRJnWajBBWn4UgliTRQ TWITTER_CALLBACK=http://127.0.0.1:8537/on/twitter/associate BOUNTYSOURCE_API_SECRET=e2BbqjNY60kC7V-Uq1dv2oHgGavbWm9pUJmiRHCApFZHDiY9aZyAspInhZaZ94x9 -BOUNTYSOURCE_API_HOST=https://staging-qa.bountysource.com -BOUNTYSOURCE_WWW_HOST=https://staging.bountysource.com BOUNTYSOURCE_CALLBACK=http://127.0.0.1:8537/on/bountysource/associate +BOUNTYSOURCE_API_HOST=https://staging-api.bountysource.com +BOUNTYSOURCE_WWW_HOST=https://staging.bountysource.com VENMO_CLIENT_ID=1534 VENMO_CLIENT_SECRET=55ckgsguYC3cj7xWW5c95PHvUzrwgZMA VENMO_CALLBACK=http://127.0.0.1:8537/on/venmo/associate -OPENSTREETMAP_API=http://master.apis.dev.openstreetmap.org OPENSTREETMAP_CONSUMER_KEY=J2SS5GM0A7tM1CIBjAHXUTMeCEkRBMYsTJzGONxe OPENSTREETMAP_CONSUMER_SECRET=hgvZkbtWVOEoaJV5AzQPcBI9m8f7BylkpT0cP7wS OPENSTREETMAP_CALLBACK=http://127.0.0.1:8537/on/openstreetmap/associate +OPENSTREETMAP_API_URL=http://master.apis.dev.openstreetmap.org/api/0.6 +OPENSTREETMAP_AUTH_URL=http://master.apis.dev.openstreetmap.org NANSWERS_THRESHOLD=2 diff --git a/gittip/billing/__init__.py b/gittip/billing/__init__.py index 68310b7c30..9d5ae83b40 100644 --- a/gittip/billing/__init__.py +++ b/gittip/billing/__init__.py @@ -19,7 +19,6 @@ from __future__ import unicode_literals from urllib import quote -import gittip import balanced import stripe from aspen.utils import typecheck diff --git a/gittip/billing/steady_state.py b/gittip/billing/steady_state.py index a983ec82d4..80f7d3b4a7 100644 --- a/gittip/billing/steady_state.py +++ b/gittip/billing/steady_state.py @@ -42,12 +42,12 @@ def converge(payouts, epsilon = 1e-10, max_rounds = 100): Converges to the steady state. """ if not issparse(payouts): - raise ArgumentError("Please provide a sparse matrix") + raise ValueError("Please provide a sparse matrix") (n_rows, n_cols) = payouts.shape if n_rows != n_cols: - raise ArgumentError("The payout matrix must be square") + raise ValueError("The payout matrix must be square") payouts_d = lil_matrix((n_rows, n_cols)) payouts_d.setdiag(payouts.diagonal()) diff --git a/gittip/elsewhere/__init__.py b/gittip/elsewhere/__init__.py index 598c3574cc..4378a22c2d 100644 --- a/gittip/elsewhere/__init__.py +++ b/gittip/elsewhere/__init__.py @@ -1,236 +1,375 @@ """This subpackage contains functionality for working with accounts elsewhere. """ -from __future__ import print_function, unicode_literals -from collections import OrderedDict +from __future__ import division, print_function, unicode_literals -from aspen.utils import typecheck -from aspen import json +from collections import OrderedDict +from datetime import datetime +import hashlib +import json +import logging +from urllib import quote +from urlparse import urlsplit, urlunsplit +import xml.etree.ElementTree as ET + +from aspen import log, Response +from aspen.utils import to_age, utc from psycopg2 import IntegrityError +from requests_oauthlib import OAuth1Session, OAuth2Session +import xmltodict -import gittip -from gittip.exceptions import ProblemChangingUsername, UnknownPlatform +from gittip.elsewhere._extractors import not_available from gittip.utils.username import reserve_a_random_username -ACTIONS = [u'opt-in', u'connect', u'lock', u'unlock'] +ACTIONS = {'opt-in', 'connect', 'lock', 'unlock'} +PLATFORMS = 'bitbucket bountysource github openstreetmap twitter venmo'.split() -# to add a new elsewhere/platform: -# 1) add its name (also the name of its module) to this list. -# it's best to append it; this ordering is used in templates. -# 2) inherit from AccountElsewhere in the platform class -# -# platform_modules will populate the platform class automatically in configure-aspen. -platforms_ordered = ( - 'twitter', - 'github', - 'bitbucket', - 'bountysource', - 'venmo', - 'openstreetmap' -) +class UnknownAccountElsewhere(Exception): pass -# init-time key setup ensures the future ordering of platform_classes will match -# platforms_ordered, since overwriting entries will maintain their order. -platform_classes = OrderedDict([(platform, None) for platform in platforms_ordered]) - -class _RegisterPlatformMeta(type): - """Tied to AccountElsewhere to enable registration by the platform field. +class PlatformRegistry(object): + """Registry of platforms we support connecting to Gittip accounts. """ + def __init__(self, platforms): + self.__dict__ = OrderedDict((p.name, p) for p in platforms) - def __new__(cls, name, bases, dct): - c = super(_RegisterPlatformMeta, cls).__new__(cls, name, bases, dct) - - # * register the platform - # * verify it was added at init-time - # * register the subclass's json encoder with aspen - c_platform = getattr(c, 'platform') - if name == 'AccountElsewhere': - pass - elif c_platform not in platform_classes: - raise UnknownPlatform(c_platform) # has it been added to platform_classes init? - else: - platform_classes[c_platform] = c + def __contains__(self, platform): + return platform.name in self.__dict__ - # aspen's json encoder registry does not take class hierarchies into account, - # so we need to register the subclasses explicitly. - json.register_encoder(c, c.to_json_compatible_object) + def __iter__(self): + return iter(self.__dict__.values()) - return c -class AccountElsewhere(object): +class UserInfo(object): + """A simple container for a user's info. - __metaclass__ = _RegisterPlatformMeta - - platform = None # set in subclass + Accessing a non-existing attribute returns `None`. + """ - # only fields in this set will be encoded - json_encode_field_whitelist = set([ - 'id', 'is_locked', 'participant', 'platform', 'user_id', 'user_info', - ]) + def __init__(self, **kw): + self.__dict__.update(kw) - def __init__(self, db, user_id, user_info=None, existing_record=None): - """Either: - - Takes a user_id and user_info, and updates the database. + def __getattr__(self, key): + return self.__dict__.get(key, None) - Or: - - Takes a user_id and existing_record, and constructs a "model" object out of the record - """ - typecheck(user_id, (int, unicode, long), user_info, (None, dict)) - self.user_id = unicode(user_id) - self.db = db + def __setattr__(self, key, value): + if value is None: + self.__dict__.pop(key, None) + else: + self.__dict__[key] = value - if user_info is not None: - a,b,c,d = self.upsert(user_info) - self.participant = a - self.is_claimed = b - self.is_locked = c - self.balance = d +class Platform(object): - self.user_info = user_info + # "x" stands for "extract" + x_user_info = not_available + x_user_id = not_available + x_user_name = not_available + x_display_name = not_available + x_email = not_available + x_gravatar_id = not_available + x_avatar_url = not_available + x_is_team = not_available - # hack to make this into a weird pseudo-model that can share convenience methods - elif existing_record is not None: - self.participant = existing_record.participant - self.is_claimed, self.is_locked, self.balance = self.get_misc_info(self.participant) - self.user_info = existing_record.user_info - self.record = existing_record + required_attrs = ( 'account_url' + , 'display_name' + , 'icon' + , 'name' + ) - def to_json_compatible_object(self): + def __init__(self, db, api_key, api_secret, callback_url, api_url=None, auth_url=None): + self.db = db + self.api_key = api_key + self.api_secret = api_secret + self.callback_url = callback_url + if api_url: + self.api_url = api_url + if auth_url: + self.auth_url = auth_url + elif not getattr(self, 'auth_url', None): + self.auth_url = self.api_url + + # Determine the appropriate response parser using `self.api_format` + api_format = getattr(self, 'api_format', None) + if api_format == 'json': + self.api_parser = lambda r: r.json() + elif api_format == 'xml': + self.api_parser = lambda r: ET.fromstring(r.content) + elif api_format: + raise ValueError('unknown API format: '+str(api_format)) + + # Make sure the subclass was implemented properly. + missing_attrs = [a for a in self.required_attrs if not hasattr(self, a)] + if missing_attrs: + msg = "The class %s is missing these required attributes: %s" + msg %= self.__class__.__name__, ', '.join(missing_attrs) + raise AttributeError(msg) + + def api_get(self, path, sess=None, **kw): """ - This is registered as an aspen.json encoder in configure-aspen - for all subclasses of this class. + Given a `path` (e.g. /users/foo), this function sends a GET request to + the platform's API (e.g. https://api.github.com/users/foo). - It only exports fields in the whitelist. + The response is returned, after checking its status code and ratelimit + headers. """ - output = {k: v for (k,v) in self.record._asdict().items() - if k in self.json_encode_field_whitelist} - - return output - - def set_is_locked(self, is_locked): - self.db.run(""" - - UPDATE elsewhere - SET is_locked=%s - WHERE platform=%s AND user_id=%s - - """, (is_locked, self.platform, self.user_id)) - + if not sess: + sess = self.get_auth_session() + response = sess.get(self.api_url+path, **kw) + + # Check status + status = response.status_code + if status == 404: + raise Response(404) + elif status != 200: + log('{} api responded with {}:\n{}'.format(self.name, status, response.text) + , level=logging.ERROR) + raise Response(500, '{} lookup failed with {}'.format(self.name, status)) + + # Check ratelimit headers + prefix = getattr(self, 'ratelimit_headers_prefix', None) + if prefix: + limit = response.headers[prefix+'limit'] + remaining = response.headers[prefix+'remaining'] + reset = response.headers[prefix+'reset'] + try: + limit, remaining, reset = int(limit), int(remaining), int(reset) + except (TypeError, ValueError): + d = dict(limit=limit, remaining=remaining, reset=reset) + log('Got weird rate headers from %s: %s' % (self.name, d)) + else: + percent_remaining = remaining/limit + if percent_remaining < 0.5: + reset = to_age(datetime.fromtimestamp(reset, tz=utc)) + log_msg = ( + '{0} API: {1:.1%} of ratelimit has been consumed, ' + '{2} requests remaining, resets {3}.' + ).format(self.name, 1 - percent_remaining, remaining, reset) + log_lvl = logging.WARNING + if percent_remaining < 0.2: + log_lvl = logging.ERROR + elif percent_remaining < 0.05: + log_lvl = logging.CRITICAL + log(log_msg, log_lvl) + + return response + + def extract_user_info(self, info): + """ + Given a user_info object of variable type (depending on the platform), + extract the relevant information by calling the platform's extractors + (`x_user_name`, `x_user_id`, etc). - def opt_in(self, desired_username): - """Given a desired username, return a User object. + Returns a `UserInfo`. The `user_id` and `user_name` attributes are + guaranteed to have non-empty values. """ - from gittip.security.user import User - - self.set_is_locked(False) - user = User.from_username(self.participant) - user.sign_in() - assert not user.ANON, self.participant # sanity check - if self.is_claimed: - newly_claimed = False + r = UserInfo() + info = self.x_user_info(info, info) + r.user_name = self.x_user_name(info) + if self.x_user_id.__func__ is not_available: + r.user_id = r.user_name else: - newly_claimed = True - user.participant.set_as_claimed() - try: - user.participant.change_username(desired_username) - except ProblemChangingUsername: - pass - return user, newly_claimed - + r.user_id = self.x_user_id(info) + assert r.user_id is not None + r.user_id = unicode(r.user_id) + assert len(r.user_id) > 0 + r.display_name = self.x_display_name(info, None) + r.email = self.x_email(info, None) + gravatar_id = self.x_gravatar_id(info, None) + if r.email and not gravatar_id: + gravatar_id = hashlib.md5(r.email.strip().lower()).hexdigest() + if gravatar_id: + r.avatar_url = 'https://www.gravatar.com/avatar/'+gravatar_id + else: + r.avatar_url = self.x_avatar_url(info, None) + r.is_team = self.x_is_team(info, False) + r.extra_info = info + return r - def upsert(self, user_info): - """Given a dict, return a tuple. + def get_account(self, user_name): + """Given a user_name on the platform, return an AccountElsewhere object. + """ + try: + return self.get_account_from_db(user_name) + except UnknownAccountElsewhere: + return self.get_account_from_api(user_name) - User_id is an immutable unique identifier for the given user on the - given platform. Username is the user's login/username on the given - platform. It is only used here for logging. Specifically, we don't - reserve their username for them on Gittip if they're new here. We give - them a random username here, and they'll have a chance to change it - if/when they opt in. User_id and username may or may not be the same. - User_info is a dictionary of profile info per the named platform. All - platform dicts must have an id key that corresponds to the primary key - in the underlying table in our own db. + def get_account_from_api(self, user_name): + """Given a user_name on the platform, get the user's info from the API, + insert it into the database, and return an AccountElsewhere object. + """ + return self.upsert(self.get_user_info(user_name)) - The return value is a tuple: (username [unicode], is_claimed [boolean], - is_locked [boolean], balance [Decimal]). + def get_account_from_db(self, user_name): + """Given a user_name on the platform, return an AccountElsewhere object. + If the account is unknown to us, we raise UnknownAccountElsewhere. """ - typecheck(user_info, dict) + exception = UnknownAccountElsewhere(self.name, user_name) + return self.db.one(""" + SELECT elsewhere.*::elsewhere_with_participant + FROM elsewhere + WHERE platform = %s + AND user_name = %s - # Insert the account if needed. - # ============================= - # Do this with a transaction so that if the insert fails, the - # participant we reserved for them is rolled back as well. + """, (self.name, user_name), default=exception) + def get_team_members(self, team_name, page_url=None): + """Given a team_name on the platform, get the team's membership list + from the API and return corresponding `AccountElsewhere`s. + """ + default_url = self.api_team_members_path.format(user_name=quote(team_name)) + r = self.api_get(page_url or default_url) + members, count, pages_urls = self.api_paginator(r, self.api_parser(r)) + members = [self.extract_user_info(m) for m in members] + accounts = self.db.all("""\ + + SELECT elsewhere.*::elsewhere_with_participant + FROM elsewhere + WHERE platform = %s + AND user_name = any(%s) + + """, (self.name, [m.user_name for m in members])) + found_user_names = set(a.user_name for a in accounts) + for member in members: + if member.user_name not in found_user_names: + accounts.append(self.upsert(member)) + return accounts, count, pages_urls + + def get_user_info(self, user_name, sess=None): + """Given a user_name on the platform, get the user's info from the API. + """ try: - with self.db.get_cursor() as cursor: - _username = reserve_a_random_username(cursor) - cursor.execute( "INSERT INTO elsewhere " - "(platform, user_id, participant) " - "VALUES (%s, %s, %s)" - , (self.platform, self.user_id, _username) - ) - except IntegrityError: - pass - - - # Update their user_info. - # ======================= - # Cast everything to unicode, because (I believe) hstore can take any - # type of value, but psycopg2 can't. - # - # https://postgres.heroku.com/blog/past/2012/3/14/introducing_keyvalue_data_storage_in_heroku_postgres/ - # http://initd.org/psycopg/docs/extras.html#hstore-data-type - # - # XXX This clobbers things, of course, such as booleans. See - # /on/bitbucket/%username/index.html - - for k, v in user_info.items(): - user_info[k] = unicode(v) - - - username = self.db.one(""" + path = self.api_user_info_path.format(user_name=quote(user_name)) + except KeyError: + raise Response(404) + info = self.api_parser(self.api_get(path, sess=sess)) + return self.extract_user_info(info) + + def get_user_self_info(self, sess): + """Get the authenticated user's info from the API. + """ + r = self.api_get(self.api_user_self_info_path, sess=sess) + return self.extract_user_info(self.api_parser(r)) + def save_token(self, user_id, token, refresh_token=None, expires=None): + """Saves the given access token in the database. + """ + self.db.run(""" UPDATE elsewhere - SET user_info=%s - WHERE platform=%s AND user_id=%s - RETURNING participant - - """, (user_info, self.platform, self.user_id)) - - return (username,) + self.get_misc_info(username) - - def get_misc_info(self, username): - rec = self.db.one(""" - - SELECT claimed_time, balance, is_locked - FROM participants - JOIN elsewhere - ON participants.username=participant - WHERE platform=%s - AND participants.username=%s + SET (access_token, refresh_token, expires) = (%s, %s, %s) + WHERE platform=%s AND user_id=%s + """, (token, refresh_token, expires, self.name, user_id)) - """, (self.platform, username)) + def upsert(self, i): + """Insert or update the user's info. + """ - assert rec is not None # sanity check + # Clean up avatar_url + if i.avatar_url: + scheme, netloc, path, query, fragment = urlsplit(i.avatar_url) + fragment = '' + if netloc.endswith('gravatar.com'): + query = 's=128' + i.avatar_url = urlunsplit((scheme, netloc, path, query, fragment)) - return ( rec.claimed_time is not None - , rec.is_locked - , rec.balance - ) + # Serialize extra_info + if isinstance(i.extra_info, ET.Element): + i.extra_info = xmltodict.parse(ET.tostring(i.extra_info)) + i.extra_info = json.dumps(i.extra_info) - def set_oauth_tokens(self, access_token, refresh_token, expires): - """ - Updates the elsewhere row with the given access token, refresh token, and Python datetime - """ + cols, vals = zip(*i.__dict__.items()) + cols = ', '.join(cols) + placeholders = ', '.join(['%s']*len(vals)) + try: + # Try to insert the account + # We do this with a transaction so that if the insert fails, the + # participant we reserved for them is rolled back as well. + with self.db.get_cursor() as cursor: + username = reserve_a_random_username(cursor) + cursor.execute(""" + INSERT INTO elsewhere + (participant, platform, {0}) + VALUES (%s, %s, {1}) + """.format(cols, placeholders), (username, self.name)+vals) + except IntegrityError: + # The account is already in the DB, update it instead + username = self.db.one(""" + UPDATE elsewhere + SET ({0}) = ({1}) + WHERE platform=%s AND user_id=%s + RETURNING participant + """.format(cols, placeholders), vals+(self.name, i.user_id)) + + # Propagate avatar_url to participant self.db.run(""" - UPDATE elsewhere - SET (access_token, refresh_token, expires) - = (%s, %s, %s) - WHERE platform=%s AND user_id=%s - """, (access_token, refresh_token, expires, self.platform, self.user_id)) + UPDATE participants p + SET avatar_url = ( + SELECT avatar_url + FROM elsewhere + WHERE participant = p.username + ORDER BY platform = 'github' DESC, + avatar_url LIKE '%%gravatar.com%%' DESC + LIMIT 1 + ) + WHERE p.username = %s + """, (username,)) + + # Now delegate to get_account_from_db + return self.get_account_from_db(i.user_name) + + +class PlatformOAuth1(Platform): + + request_token_path = '/oauth/request_token' + authorize_path = '/oauth/authorize' + access_token_path = '/oauth/access_token' + + def get_auth_session(self, token=None, token_secret=None): + return OAuth1Session(self.api_key, self.api_secret, token, token_secret, + callback_uri=self.callback_url) + + def get_auth_url(self, **kw): + sess = self.get_auth_session() + r = sess.fetch_request_token(self.auth_url+self.request_token_path) + url = sess.authorization_url(self.auth_url+self.authorize_path) + return url, r['oauth_token'], r['oauth_token_secret'] + + def get_query_id(self, querystring): + return querystring['oauth_token'] + + def handle_auth_callback(self, url, token, token_secret): + sess = self.get_auth_session(token=token, token_secret=token_secret) + sess.parse_authorization_response(url) + sess.fetch_access_token(self.auth_url+self.access_token_path) + return sess + + +class PlatformOAuth2(Platform): + + oauth_default_scope = None + oauth_email_scope = None + oauth_payment_scope = None + + def get_auth_session(self, state=None, token=None): + return OAuth2Session(self.api_key, state=state, token=token, + redirect_uri=self.callback_url, + scope=self.oauth_default_scope) + + def get_auth_url(self, **kw): + sess = self.get_auth_session() + url, state = sess.authorization_url(self.auth_url+'/authorize') + return url, state, '' + + def get_query_id(self, querystring): + return querystring['state'] + + def handle_auth_callback(self, url, state, unused_arg): + sess = self.get_auth_session(state=state) + sess.fetch_token(self.auth_url+'/access_token', + client_secret=self.api_secret, + authorization_response=url) + return sess diff --git a/gittip/elsewhere/_extractors.py b/gittip/elsewhere/_extractors.py new file mode 100644 index 0000000000..4b021638e1 --- /dev/null +++ b/gittip/elsewhere/_extractors.py @@ -0,0 +1,66 @@ +"""Helper functions to extract data from API responses +""" +from __future__ import unicode_literals + +import json +import xml.etree.ElementTree as ET + +from aspen import log + + +def key(k, clean=lambda a: a): + def f(self, info, *default): + try: + v = info.pop(k, *default) + except KeyError: + msg = 'Unable to find key "%s" in %s API response:\n%s' + log(msg % (k, self.name, json.dumps(info, indent=4))) + raise + if v: + v = clean(v) + if not v and not default: + msg = 'Key "%s" has an empty value in %s API response:\n%s' + msg %= (k, self.name, json.dumps(info, indent=4)) + log(msg) + raise ValueError(msg) + return v + return f + + +def not_available(self, info, default): + return default + + +def xpath(path, attr=None, clean=lambda a: a): + def f(self, info, *default): + try: + l = info.findall(path) + if len(l) > 1: + msg = 'The xpath "%s" matches more than one element in %s API response:\n%s' + msg %= (path, self.name, ET.tostring(info)) + log(msg) + raise ValueError(msg) + v = l[0].get(attr) if attr else l[0] + except IndexError: + if default: + return default[0] + msg = 'Unable to find xpath "%s" in %s API response:\n%s' + msg %= (path, self.name, ET.tostring(info)) + log(msg) + raise IndexError(msg) + except KeyError: + if default: + return default[0] + msg = 'The element has no "%s" attribute in %s API response:\n%s' + msg %= (attr, self.name, ET.tostring(info)) + log(msg) + raise KeyError(msg) + if v: + v = clean(v) + if not v and not default: + msg = 'The xpath "%s" points to an empty value in %s API response:\n%s' + msg %= (path, self.name, ET.tostring(info)) + log(msg) + raise ValueError(msg) + return v + return f diff --git a/gittip/elsewhere/_paginators.py b/gittip/elsewhere/_paginators.py new file mode 100644 index 0000000000..88b3fe7de2 --- /dev/null +++ b/gittip/elsewhere/_paginators.py @@ -0,0 +1,36 @@ +"""Helper functions to handle pagination of API responses +""" +from __future__ import unicode_literals + + +def _relativize_urls(base, urls): + i = len(base) + r = {} + for link_key, url in urls.items(): + if not url.startswith(base): + raise ValueError('"%s" is not a prefix of "%s"' % (base, url)) + r[link_key] = url[i:] + return r + + +links_keys = set('prev next first last'.split()) + + +def header_links_paginator(): + def f(self, response, parsed): + links = {k: v['url'] for k, v in response.links.items() if k in links_keys} + total_count = -1 if links else len(parsed) + return parsed, total_count, _relativize_urls(self.api_url, links) + return f + + +def keys_paginator(**kw): + page_key = kw.get('page', 'values') + total_count_key = kw.get('total_count', 'size') + links_keys_map = tuple((k, kw.get(k, k)) for k in links_keys) + def f(self, response, parsed): + page = parsed[page_key] + links = {k: parsed[k2] for k, k2 in links_keys_map if parsed.get(k2)} + total_count = parsed.get(total_count_key, -1) if links else len(page) + return page, total_count, _relativize_urls(self.api_url, links) + return f diff --git a/gittip/elsewhere/bitbucket.py b/gittip/elsewhere/bitbucket.py index 2f0623811c..91de5d343b 100644 --- a/gittip/elsewhere/bitbucket.py +++ b/gittip/elsewhere/bitbucket.py @@ -1,74 +1,35 @@ -import logging - -import gittip -import requests -from aspen import json, log, Response -from aspen.http.request import PathPart -from aspen.utils import typecheck -from gittip.elsewhere import AccountElsewhere - - -BASE_API_URL = "https://bitbucket.org/api/1.0" - - -class BitbucketAccount(AccountElsewhere): - platform = u'bitbucket' - - def get_url(self): - url = "https://bitbucket.org/%s" % self.user_info["username"] - return url - - def get_user_name(self): - return self.user_info['username'] - - def get_platform_icon(self): - return "/assets/icons/bitbucket.12.png" - - -def oauth_url(website, action, then=""): - """Return a URL to start oauth dancing with Bitbucket. - - For GitHub we can pass action and then through a querystring. For Bitbucket - we can't, so we send people through a local URL first where we stash this - info in an in-memory cache (eep! needs refactoring to scale). - - Not sure why website is here. Vestige from GitHub forebear? - - """ - then = then.encode('base64').strip() - return "/on/bitbucket/redirect?action=%s&then=%s" % (action, then) - - -def get_user_info(db, username): - """Get the given user's information from the DB or failing that, bitbucket. - - :param username: - A unicode string representing a username in bitbucket. - - :returns: - A dictionary containing bitbucket specific information for the user. - """ - typecheck(username, (unicode, PathPart)) - rec = db.one(""" - SELECT user_info FROM elsewhere - WHERE platform='bitbucket' - AND user_info->'username' = %s - """, (username,)) - if rec is not None: - user_info = rec - else: - url = "%s/users/%s?pagelen=100" - user_info = requests.get(url % (BASE_API_URL, username)) - status = user_info.status_code - content = user_info.content - if status == 200: - user_info = json.loads(content)['user'] - elif status == 404: - raise Response(404, - "Bitbucket identity '{0}' not found.".format(username)) - else: - log("Bitbucket api responded with {0}: {1}".format(status, content), - level=logging.WARNING) - raise Response(502, "Bitbucket lookup failed with %d." % status) - - return user_info +from __future__ import absolute_import, division, print_function, unicode_literals + +from gittip.elsewhere import PlatformOAuth1 +from gittip.elsewhere._extractors import key, not_available +from gittip.elsewhere._paginators import keys_paginator + + +class Bitbucket(PlatformOAuth1): + + # Platform attributes + name = 'bitbucket' + display_name = 'Bitbucket' + account_url = 'https://bitbucket.org/{user_name}' + icon = '/assets/icons/bitbucket.12.png' + + # Auth attributes + auth_url = 'https://bitbucket.org/api/1.0' + authorize_path = '/oauth/authenticate' + + # API attributes + api_format = 'json' + api_paginator = keys_paginator(prev='previous') + api_url = 'https://bitbucket.org/api' + api_user_info_path = '/1.0/users/{user_name}' + api_user_self_info_path = '/1.0/user' + api_team_members_path = '/2.0/teams/{user_name}/members' + + # User info extractors + x_user_info = key('user') + x_user_id = not_available # No immutable id. :-/ + x_user_name = key('username') + x_display_name = key('display_name') + x_email = not_available + x_avatar_url = key('avatar') + x_is_team = key('is_team') diff --git a/gittip/elsewhere/bountysource.py b/gittip/elsewhere/bountysource.py index 62930c4ff8..a21a633ab0 100644 --- a/gittip/elsewhere/bountysource.py +++ b/gittip/elsewhere/bountysource.py @@ -1,115 +1,84 @@ -import os -import md5 -import time -from gittip.models.participant import Participant -from gittip.elsewhere import AccountElsewhere - - -class BountysourceAccount(AccountElsewhere): - platform = u'bountysource' - - def get_url(self): - - # I don't see that we actually use this. Leaving as a stub pending - # https://github.com/gittip/www.gittip.com/pull/1369. - - raise NotImplementedError - - def get_user_name(self): - return self.user_info['display_name'] - - def get_platform_icon(self): - return "/assets/icons/bountysource.12.png" - - -def oauth_url(website, participant, redirect_url=None): - """Return a URL to authenticate with Bountysource. - - :param participant: - The participant whose account is being linked - - :param redirect_url: - Optional redirect URL after authentication. Defaults to value defined - in local.env - - :returns: - URL for Bountysource account authorization - """ - if redirect_url: - return "/on/bountysource/redirect?redirect_url=%s" % redirect_url - else: - return "/on/bountysource/redirect" - - -# Bountysource Access Tokens -# ========================== - -def create_access_token(participant): - """Return an access token for the Bountysource API for this user. - """ - time_now = int(time.time()) - token = "%s.%s.%s" % ( participant.id - , time_now - , hash_access_token(participant.id, time_now) - ) - return token - - -def hash_access_token(user_id, time_now): - """Create hash for access token. - - :param user_id: - ID of the user. - - :param time_now: - Current time, in seconds, as an integer. - - :returns: - MD5 hash of user_id, time, and Bountysource API secret - """ - raw = "%s.%s.%s" % ( user_id - , time_now - , os.environ['BOUNTYSOURCE_API_SECRET'].decode('ASCII') - ) - return md5.new(raw).hexdigest() - - -def access_token_valid(access_token): - """Helper method to check validity of access token. - """ - parts = (access_token or '').split('.') - return len(parts) == 3 and parts[2] == \ - hash_access_token(parts[0], parts[1]) - - -def get_participant_via_access_token(access_token): - """From a Gittip access token, attempt to find an external account - - :param access_token: - access token generated by Gittip on account link redirect - - :returns: - the participant, if found - """ - if access_token_valid(access_token): - parts = access_token.split('.') - participant_id = parts[0] - return Participant.from_id(participant_id) - - -def filter_user_info(user_info): - """Filter the user info dictionary for a Bountysource account. - - This is so that the Bountysource access token doesn't float around in a - user_info hash (considering nothing else does that). - - """ - whitelist = ['id', 'display_name', 'first_name', 'last_name', 'email', \ - 'frontend_url', 'image_url'] - filtered_user_info = {} - for key in user_info: - if key in whitelist: - filtered_user_info[key] = user_info[key] - - return filtered_user_info +from __future__ import absolute_import, division, print_function, unicode_literals +from binascii import hexlify +import hashlib +import os +from time import time +from urllib import urlencode +from urlparse import parse_qs, urlparse + +import requests + +from aspen import Response +from gittip.elsewhere import Platform +from gittip.elsewhere._extractors import key, not_available + + +class Bountysource(Platform): + + # Platform attributes + name = 'bountysource' + display_name = 'Bountysource' + account_url = '{platform_data.auth_url}/people/{user_id}' + icon = '/assets/icons/bountysource.12.png' + + # API attributes + api_format = 'json' + api_user_info_path = '/users/{user_id}' + api_user_self_info_path = '/user' + + # User info extractors + x_user_id = key('id') + x_user_name = key('display_name') + x_display_name = not_available + x_email = key('email') + x_avatar_url = key('image_url') + + def get_auth_session(self, token=None): + sess = requests.Session() + sess.auth = BountysourceAuth(token) + return sess + + def get_auth_url(self, user): + query_id = hexlify(os.urandom(10)) + time_now = int(time()) + raw = '%s.%s.%s' % (user.participant.id, time_now, self.api_secret) + h = hashlib.md5(raw).hexdigest() + token = '%s.%s.%s' % (user.participant.id, time_now, h) + params = dict( + redirect_url=self.callback_url+'?query_id='+query_id, + external_access_token=token + ) + url = self.auth_url+'/auth/gittip/confirm?'+urlencode(params) + return url, query_id, '' + + def get_query_id(self, querystring): + token = querystring['access_token'] + i = token.rfind('.') + data, data_hash = token[:i], token[i+1:] + if data_hash != hashlib.md5(data+'.'+self.api_secret).hexdigest(): + raise Response(400, 'Invalid hash in access_token') + return querystring['query_id'] + + def get_user_self_info(self, sess): + querystring = urlparse(sess._callback_url).query + info = {k: v[0] if len(v) == 1 else v + for k, v in parse_qs(querystring).items()} + info.pop('access_token') + info.pop('query_id') + return self.extract_user_info(info) + + def handle_auth_callback(self, url, query_id, unused_arg): + sess = self.get_auth_session(token=query_id) + sess._callback_url=url + return sess + + +class BountysourceAuth(object): + + def __init__(self, token=None): + self.token = token + + def __call__(self, req): + if self.token: + req.params['access_token'] = self.token diff --git a/gittip/elsewhere/github.py b/gittip/elsewhere/github.py index c3b05e6a70..df657d17b1 100644 --- a/gittip/elsewhere/github.py +++ b/gittip/elsewhere/github.py @@ -1,149 +1,36 @@ -from __future__ import division -import gittip -import logging -import requests -import os -from aspen import json, Response -from aspen.http.request import PathPart -from aspen.utils import typecheck -from aspen.website import Website -from gittip import log -from gittip.elsewhere import ACTIONS, AccountElsewhere - - -class GitHubAccount(AccountElsewhere): - platform = u'github' - - def get_url(self): - return self.user_info['html_url'] - - def get_user_name(self): - return self.user_info['login'] - - def get_platform_icon(self): - return "/assets/icons/github.12.png" - - -def oauth_url(website, action, then=u""): - """Given a website object and a string, return a URL string. - - 'action' is one of 'opt-in', 'lock' and 'unlock' - - 'then' is either a github username or an URL starting with '/'. It's - where we'll send the user after we get the redirect back from - GitHub. - - """ - typecheck(website, Website, action, unicode, then, unicode) - assert action in ACTIONS - url = u"https://github.com/login/oauth/authorize?client_id=%s&redirect_uri=%s" - url %= (website.github_client_id, website.github_callback) - - # Pack action,then into data and base64-encode. Querystring isn't - # available because it's consumed by the initial GitHub request. - - data = u'%s,%s' % (action, then) - data = data.encode('UTF-8').encode('base64').strip().decode('US-ASCII') - url += u'?data=%s' % data - return url - - -def oauth_dance(website, qs): - """Given a querystring, return a dict of user_info. - - The querystring should be the querystring that we get from GitHub when - we send the user to the return value of oauth_url above. - - See also: - - http://developer.github.com/v3/oauth/ - - """ - - log("Doing an OAuth dance with Github.") - - data = { 'code': qs['code'].encode('US-ASCII') - , 'client_id': website.github_client_id - , 'client_secret': website.github_client_secret - } - r = requests.post("https://github.com/login/oauth/access_token", data=data) - assert r.status_code == 200, (r.status_code, r.text) - - back = dict([pair.split('=') for pair in r.text.split('&')]) # XXX - if 'error' in back: - raise Response(400, back['error'].encode('utf-8')) - assert back.get('token_type', '') == 'bearer', back - access_token = back['access_token'] - - r = requests.get( "https://api.github.com/user" - , headers={'Authorization': 'token %s' % access_token} - ) - assert r.status_code == 200, (r.status_code, r.text) - user_info = json.loads(r.text) - log("Done with OAuth dance with Github for %s (%s)." - % (user_info['login'], user_info['id'])) - - return user_info - - -def get_user_info(db, login): - """Get the given user's information from the DB or failing that, github. - - :param login: - A unicode string representing a username in github. - - :returns: - A dictionary containing github specific information for the user. - """ - typecheck(login, (unicode, PathPart)) - rec = db.one(""" - SELECT user_info FROM elsewhere - WHERE platform='github' - AND user_info->'login' = %s - """, (login,)) - - if rec is not None: - user_info = rec - else: - url = "https://api.github.com/users/%s" - user_info = requests.get(url % login, params={ - 'client_id': os.environ.get('GITHUB_CLIENT_ID'), - 'client_secret': os.environ.get('GITHUB_CLIENT_SECRET') - }) - status = user_info.status_code - content = user_info.text - - # Calculate how much of our ratelimit we have consumed - remaining = int(user_info.headers['x-ratelimit-remaining']) - limit = int(user_info.headers['x-ratelimit-limit']) - # thanks to from __future__ import division this is a float - percent_remaining = remaining/limit - - log_msg = '' - log_lvl = None - # We want anything 50% or over - if 0.5 <= percent_remaining: - log_msg = ("{0}% of GitHub's ratelimit has been consumed. {1}" - " requests remaining.").format(percent_remaining * 100, - remaining) - if 0.5 <= percent_remaining < 0.8: - log_lvl = logging.WARNING - elif 0.8 <= percent_remaining < 0.95: - log_lvl = logging.ERROR - elif 0.95 <= percent_remaining: - log_lvl = logging.CRITICAL - - if log_msg and log_lvl: - log(log_msg, log_lvl) - - if status == 200: - user_info = json.loads(content) - elif status == 404: - raise Response(404, - "GitHub identity '{0}' not found.".format(login)) - else: - log("Github api responded with {0}: {1}".format(status, content), - level=logging.WARNING) - raise Response(502, "GitHub lookup failed with %d." % status) - - return user_info +from __future__ import absolute_import, division, print_function, unicode_literals + +from gittip.elsewhere import PlatformOAuth2 +from gittip.elsewhere._extractors import key +from gittip.elsewhere._paginators import header_links_paginator + + +class GitHub(PlatformOAuth2): + + # Platform attributes + name = 'github' + display_name = 'GitHub' + account_url = 'https://github.com/{user_name}' + icon = '/assets/icons/github.12.png' + + # Auth attributes + auth_url = 'https://github.com/login/oauth' + oauth_email_scope = 'user:email' + + # API attributes + api_format = 'json' + api_paginator = header_links_paginator() + api_url = 'https://api.github.com' + api_user_info_path = '/users/{user_name}' + api_user_self_info_path = '/user' + api_team_members_path = '/orgs/{user_name}/public_members' + ratelimit_headers_prefix = 'x-ratelimit-' + + # User info extractors + x_user_id = key('id') + x_user_name = key('login') + x_display_name = key('name') + x_email = key('email') + x_gravatar_id = key('gravatar_id') + x_avatar_url = key('avatar_url') + x_is_team = key('type', clean=lambda t: t.lower() == 'organization') diff --git a/gittip/elsewhere/openstreetmap.py b/gittip/elsewhere/openstreetmap.py index 7bb8116f6d..477c418d4e 100644 --- a/gittip/elsewhere/openstreetmap.py +++ b/gittip/elsewhere/openstreetmap.py @@ -1,72 +1,25 @@ -import logging +from __future__ import absolute_import, division, print_function, unicode_literals -import gittip -import requests -from aspen import json, log, Response -from aspen.http.request import PathPart -from aspen.utils import typecheck -from gittip.elsewhere import AccountElsewhere +from gittip.elsewhere import PlatformOAuth1 +from gittip.elsewhere._extractors import not_available, xpath +class OpenStreetMap(PlatformOAuth1): -class OpenStreetMapAccount(AccountElsewhere): - platform = u'openstreetmap' + # Platform attributes + name = 'openstreetmap' + display_name = 'OpenStreetMap' + account_url = 'http://www.openstreetmap.org/user/{user_name}' + icon = '/assets/icons/openstreetmap.12.png' - def get_url(self): - return self.user_info['html_url'] + # API attributes + api_format = 'xml' + api_user_info_path = '/user/{user_id}' + api_user_self_info_path = '/user/details' - def get_user_name(self): - return self.user_info['username'] - - def get_platform_icon(self): - return "/assets/icons/openstreetmap.12.png" - - -def oauth_url(website, action, then=""): - """Return a URL to start oauth dancing with OpenStreetMap. - - For GitHub we can pass action and then through a querystring. For OpenStreetMap - we can't, so we send people through a local URL first where we stash this - info in an in-memory cache (eep! needs refactoring to scale). - - Not sure why website is here. Vestige from GitHub forebear? - - """ - then = then.encode('base64').strip() - return "/on/openstreetmap/redirect?action=%s&then=%s" % (action, then) - - -def get_user_info(db, username, osm_api_url): - """Get the given user's information from the DB or failing that, openstreetmap. - - :param username: - A unicode string representing a username in OpenStreetMap. - - :param osm_api_url: - URL of OpenStreetMap API. - - :returns: - A dictionary containing OpenStreetMap specific information for the user. - """ - typecheck(username, (unicode, PathPart)) - rec = db.one(""" - SELECT user_info FROM elsewhere - WHERE platform='openstreetmap' - AND user_info->'username' = %s - """, (username,)) - if rec is not None: - user_info = rec - else: - osm_user = requests.get("%s/user/%s" % (osm_api_url, username)) - if osm_user.status_code == 200: - log("User %s found in OpenStreetMap but not in gittip." % username) - user_info = None - elif osm_user.status_code == 404: - raise Response(404, - "OpenStreetMap identity '{0}' not found.".format(username)) - else: - log("OpenStreetMap api responded with {0}: {1}".format(status, content), - level=logging.WARNING) - raise Response(502, "OpenStreetMap lookup failed with %d." % status) - - return user_info + # User info extractors + x_user_id = xpath('./user', attr='id') + x_user_name = xpath('./user', attr='display_name') + x_display_name = x_user_name + x_email = not_available + x_avatar_url = xpath('./user/img', attr='href') diff --git a/gittip/elsewhere/twitter.py b/gittip/elsewhere/twitter.py index c2d28c882c..042b919db8 100644 --- a/gittip/elsewhere/twitter.py +++ b/gittip/elsewhere/twitter.py @@ -1,95 +1,31 @@ -import datetime -import gittip -import requests -from aspen import json, log, Response -from aspen.http.request import PathPart -from aspen.utils import to_age, utc, typecheck -from gittip.elsewhere import AccountElsewhere -from os import environ -from requests_oauthlib import OAuth1 +from __future__ import absolute_import, division, print_function, unicode_literals +from gittip.elsewhere import PlatformOAuth1 +from gittip.elsewhere._extractors import key, not_available -class TwitterAccount(AccountElsewhere): - platform = u'twitter' - def get_url(self): - return "https://twitter.com/" + self.user_info['screen_name'] +class Twitter(PlatformOAuth1): - def get_user_name(self): - return self.user_info['screen_name'] + # Platform attributes + name = 'twitter' + display_name = 'Twitter' + account_url = 'https://twitter.com/{user_name}' + icon = '/assets/icons/twitter.12.png' - def get_platform_icon(self): - return "/assets/icons/twitter.12.png" + # Auth attributes + auth_url = 'https://api.twitter.com' + # API attributes + api_format = 'json' + api_url = 'https://api.twitter.com/1.1' + api_user_info_path = '/users/show.json?screen_name={user_name}' + api_user_self_info_path = '/account/verify_credentials.json' + ratelimit_headers_prefix = 'x-rate-limit-' -def oauth_url(website, action, then=""): - """Return a URL to start oauth dancing with Twitter. - - For GitHub we can pass action and then through a querystring. For Twitter - we can't, so we send people through a local URL first where we stash this - info in an in-memory cache (eep! needs refactoring to scale). - - Not sure why website is here. Vestige from GitHub forebear? - - """ - then = then.encode('base64').strip() - return "/on/twitter/redirect?action=%s&then=%s" % (action, then) - - -def get_user_info(db, screen_name): - """Given a unicode, return a dict. - """ - typecheck(screen_name, (unicode, PathPart)) - rec = db.one(""" - SELECT user_info FROM elsewhere - WHERE platform='twitter' - AND user_info->'screen_name' = %s - """, (screen_name,)) - - if rec is not None: - user_info = rec - else: - # Updated using Twython as a point of reference: - # https://github.com/ryanmcgrath/twython/blob/master/twython/twython.py#L76 - oauth = OAuth1( - # we do not have access to the website obj, - # so let's grab the details from the env - environ['TWITTER_CONSUMER_KEY'], - environ['TWITTER_CONSUMER_SECRET'], - environ['TWITTER_ACCESS_TOKEN'], - environ['TWITTER_ACCESS_TOKEN_SECRET'], - ) - - url = "https://api.twitter.com/1.1/users/show.json?screen_name=%s" - user_info = requests.get(url % screen_name, auth=oauth) - - # Keep an eye on our Twitter usage. - # ================================= - - rate_limit = user_info.headers['X-Rate-Limit-Limit'] - rate_limit_remaining = user_info.headers['X-Rate-Limit-Remaining'] - rate_limit_reset = user_info.headers['X-Rate-Limit-Reset'] - - try: - rate_limit = int(rate_limit) - rate_limit_remaining = int(rate_limit_remaining) - rate_limit_reset = int(rate_limit_reset) - except (TypeError, ValueError): - log( "Got weird rate headers from Twitter: %s %s %s" - % (rate_limit, rate_limit_remaining, rate_limit_reset) - ) - else: - reset = datetime.datetime.fromtimestamp(rate_limit_reset, tz=utc) - reset = to_age(reset) - log( "Twitter API calls used: %d / %d. Resets %s." - % (rate_limit - rate_limit_remaining, rate_limit, reset) - ) - - - if user_info.status_code == 200: - user_info = json.loads(user_info.text) - else: - log("Twitter lookup failed with %d." % user_info.status_code) - raise Response(404) - - return user_info + # User info extractors + x_user_id = key('id') + x_user_name = key('screen_name') + x_display_name = key('name') + x_email = not_available + x_avatar_url = key('profile_image_url_https', + clean=lambda v: v.replace('_normal.', '.')) diff --git a/gittip/elsewhere/venmo.py b/gittip/elsewhere/venmo.py index 7fc44a3af7..dee4419774 100644 --- a/gittip/elsewhere/venmo.py +++ b/gittip/elsewhere/venmo.py @@ -1,56 +1,33 @@ -from gittip.elsewhere import AccountElsewhere -from urllib import urlencode -from aspen import json, Response -import requests - - -class VenmoAccount(AccountElsewhere): - platform = u'venmo' - - def get_url(self): - return "https://venmo.com/" + self.user_info['username'] - - def get_profile_image(self): - return self.user_info['profile_picture_url'] - - def get_user_name(self): - return self.user_info['username'] - - def get_display_name(self): - return self.user_info['display_name'] - - def get_platform_icon(self): - return "/assets/icons/venmo.16.png" - -def oauth_url(website): - connect_params = { - 'client_id': website.venmo_client_id, - 'scope': 'make_payments', - 'redirect_uri': website.venmo_callback, - 'response_type': 'code' - } - url = u"https://api.venmo.com/v1/oauth/authorize?{}".format( - urlencode(connect_params) - ) - return url - -def oauth_dance(website, qs): - """Return a dictionary of the Venmo response. - - There's an example at: https://developer.venmo.com/docs/authentication - """ - - data = { - 'code': qs['code'].encode('US-ASCII'), - 'client_id': website.venmo_client_id, - 'client_secret': website.venmo_client_secret - } - r = requests.post('https://api.venmo.com/v1/oauth/access_token', data=data) - res_dict = r.json() - - if 'error' in res_dict: - raise Response(400, res_dict['error']['message'].encode('utf-8')) - - assert r.status_code == 200, (r.status_code, r.text) - - return res_dict +from __future__ import absolute_import, division, print_function, unicode_literals + +from gittip.elsewhere import PlatformOAuth2 +from gittip.elsewhere._extractors import key + + +class Venmo(PlatformOAuth2): + + # Platform attributes + name = 'venmo' + display_name = 'Venmo' + account_url = 'https://venmo.com/{user_name}' + icon = '/assets/icons/venmo.16.png' + + # PlatformOAuth2 attributes + auth_url = 'https://api.venmo.com/v1/oauth' + oauth_email_scope = 'access_email' + oauth_payment_scope = 'make_payments' + oauth_default_scope = ['access_profile'] + + # API attributes + api_format = 'json' + api_url = 'https://api.venmo.com/v1' + api_user_info_path = '/users/{user_id}' + api_user_self_info_path = '/me' + + # User info extractors + x_user_info = key('data', clean=lambda d: d.pop('user', d)) + x_user_id = key('id') + x_user_name = key('username') + x_display_name = key('display_name') + x_email = key('email') + x_avatar_url = key('profile_picture_url') diff --git a/gittip/exceptions.py b/gittip/exceptions.py index b6efae33f6..b69b11b207 100644 --- a/gittip/exceptions.py +++ b/gittip/exceptions.py @@ -5,9 +5,6 @@ from __future__ import print_function, unicode_literals - -class UnknownPlatform(Exception): pass - class ProblemChangingUsername(Exception): def __str__(self): return self.msg.format(self.args[0]) diff --git a/gittip/models/_mixin_elsewhere.py b/gittip/models/_mixin_elsewhere.py deleted file mode 100644 index 51c19d6aa5..0000000000 --- a/gittip/models/_mixin_elsewhere.py +++ /dev/null @@ -1,424 +0,0 @@ -import os -from collections import namedtuple - -from gittip import NotSane -from aspen.utils import typecheck -from psycopg2 import IntegrityError - -from gittip.exceptions import UnknownPlatform -from gittip.elsewhere import platform_classes -from gittip.utils.username import reserve_a_random_username, gen_random_usernames - - -# Exceptions -# ========== - -class NeedConfirmation(Exception): - """Represent the case where we need user confirmation during a merge. - - This is used in the workflow for merging one participant into another. - - """ - - def __init__(self, a, b, c): - self.other_is_a_real_participant = a - self.this_is_others_last_account_elsewhere = b - self.we_already_have_that_kind_of_account = c - self._all = (a, b, c) - - def __repr__(self): - return "" % self._all - __str__ = __repr__ - - def __eq__(self, other): - return self._all == other._all - - def __ne__(self, other): - return not self.__eq__(other) - - def __nonzero__(self): - # bool(need_confirmation) - A, B, C = self._all - return A or C - - -# Mixin -# ===== - -# note that the ordering of these fields is defined by platform_classes -AccountsTuple = namedtuple('AccountsTuple', platform_classes.keys()) - -class MixinElsewhere(object): - """We use this as a mixin for Participant, and in a hackish way on the - homepage and community pages. - - """ - - def get_accounts_elsewhere(self): - """Return an AccountsTuple of AccountElsewhere instances. - """ - - ACCOUNTS = "SELECT * FROM elsewhere WHERE participant=%s" - accounts = self.db.all(ACCOUNTS, (self.username,)) - - accounts_dict = {platform: None for platform in platform_classes} - - for account in accounts: - if account.platform not in platform_classes: - raise UnknownPlatform(account.platform) - - account_cls = platform_classes[account.platform] - accounts_dict[account.platform] = \ - account_cls(self.db, account.user_id, existing_record=account) - - return AccountsTuple(**accounts_dict) - - def get_img_src(self, size=128): - """Return a value for . - - Until we have our own profile pics, delegate. XXX Is this an attack - vector? Can someone inject this value? Don't think so, but if you make - it happen, let me know, eh? Thanks. :) - - https://www.gittip.com/security.txt - - """ - typecheck(size, int) - - src = '/assets/%s/avatar-default.gif' % os.environ['__VERSION__'] - - accounts = self.get_accounts_elsewhere() - - if accounts.github is not None: - # GitHub -> Gravatar: http://en.gravatar.com/site/implement/images/ - if 'gravatar_id' in accounts.github.user_info: - gravatar_hash = accounts.github.user_info['gravatar_id'] - src = "https://www.gravatar.com/avatar/%s.jpg?s=%s" - src %= (gravatar_hash, size) - - elif accounts.twitter is not None: - # https://dev.twitter.com/docs/api/1.1/get/users/show - if 'profile_image_url_https' in accounts.twitter.user_info: - src = accounts.twitter.user_info['profile_image_url_https'] - - # For Twitter, we don't have good control over size. The - # biggest option is 73px(?!), but that's too small. Let's go - # with the original: even though it may be huge, that's - # preferrable to guaranteed blurriness. :-/ - - src = src.replace('_normal.', '.') - - elif accounts.openstreetmap is not None: - if 'img_src' in accounts.openstreetmap.user_info: - src = accounts.openstreetmap.user_info['img_src'] - - return src - - - def take_over(self, account_elsewhere, have_confirmation=False): - """Given an AccountElsewhere and a bool, raise NeedConfirmation or return None. - - This method associates an account on another platform (GitHub, Twitter, - etc.) with the given Gittip participant. Every account elsewhere has an - associated Gittip participant account, even if its only a stub - participant (it allows us to track pledges to that account should they - ever decide to join Gittip). - - In certain circumstances, we want to present the user with a - confirmation before proceeding to reconnect the account elsewhere to - the new Gittip account; NeedConfirmation is the signal to request - confirmation. If it was the last account elsewhere connected to the old - Gittip account, then we absorb the old Gittip account into the new one, - effectively archiving the old account. - - Here's what absorbing means: - - - consolidated tips to and fro are set up for the new participant - - Amounts are summed, so if alice tips bob $1 and carl $1, and - then bob absorbs carl, then alice tips bob $2(!) and carl $0. - - And if bob tips alice $1 and carl tips alice $1, and then bob - absorbs carl, then bob tips alice $2(!) and carl tips alice $0. - - The ctime of each new consolidated tip is the older of the two - tips that are being consolidated. - - If alice tips bob $1, and alice absorbs bob, then alice tips - bob $0. - - If alice tips bob $1, and bob absorbs alice, then alice tips - bob $0. - - - all tips to and from the other participant are set to zero - - the absorbed username is released for reuse - - the absorption is recorded in an absorptions table - - This is done in one transaction. - - """ - - platform = account_elsewhere.platform - user_id = account_elsewhere.user_id - - CREATE_TEMP_TABLE_FOR_UNIQUE_TIPS = """ - - CREATE TEMP TABLE __temp_unique_tips ON COMMIT drop AS - - -- Get all the latest tips from everyone to everyone. - - SELECT DISTINCT ON (tipper, tippee) - ctime, tipper, tippee, amount - FROM tips - ORDER BY tipper, tippee, mtime DESC; - - """ - - CONSOLIDATE_TIPS_RECEIVING = """ - - -- Create a new set of tips, one for each current tip *to* either - -- the dead or the live account. If a user was tipping both the - -- dead and the live account, then we create one new combined tip - -- to the live account (via the GROUP BY and sum()). - - INSERT INTO tips (ctime, tipper, tippee, amount) - - SELECT min(ctime), tipper, %(live)s AS tippee, sum(amount) - - FROM __temp_unique_tips - - WHERE (tippee = %(dead)s OR tippee = %(live)s) - -- Include tips *to* either the dead or live account. - - AND NOT (tipper = %(dead)s OR tipper = %(live)s) - -- Don't include tips *from* the dead or live account, - -- lest we convert cross-tipping to self-tipping. - - AND amount > 0 - -- Don't include zeroed out tips, so we avoid a no-op - -- zero tip entry. - - GROUP BY tipper - - """ - - CONSOLIDATE_TIPS_GIVING = """ - - -- Create a new set of tips, one for each current tip *from* either - -- the dead or the live account. If both the dead and the live - -- account were tipping a given user, then we create one new - -- combined tip from the live account (via the GROUP BY and sum()). - - INSERT INTO tips (ctime, tipper, tippee, amount) - - SELECT min(ctime), %(live)s AS tipper, tippee, sum(amount) - - FROM __temp_unique_tips - - WHERE (tipper = %(dead)s OR tipper = %(live)s) - -- Include tips *from* either the dead or live account. - - AND NOT (tippee = %(dead)s OR tippee = %(live)s) - -- Don't include tips *to* the dead or live account, - -- lest we convert cross-tipping to self-tipping. - - AND amount > 0 - -- Don't include zeroed out tips, so we avoid a no-op - -- zero tip entry. - - GROUP BY tippee - - """ - - ZERO_OUT_OLD_TIPS_RECEIVING = """ - - INSERT INTO tips (ctime, tipper, tippee, amount) - - SELECT ctime, tipper, tippee, 0 AS amount - FROM __temp_unique_tips - WHERE tippee=%s AND amount > 0 - - """ - - ZERO_OUT_OLD_TIPS_GIVING = """ - - INSERT INTO tips (ctime, tipper, tippee, amount) - - SELECT ctime, tipper, tippee, 0 AS amount - FROM __temp_unique_tips - WHERE tipper=%s AND amount > 0 - - """ - - with self.db.get_cursor() as cursor: - - # Load the existing connection. - # ============================= - # Every account elsewhere has at least a stub participant account - # on Gittip. - - rec = cursor.one(""" - - SELECT participant - , claimed_time IS NULL AS is_stub - FROM elsewhere - JOIN participants ON participant=participants.username - WHERE elsewhere.platform=%s AND elsewhere.user_id=%s - - """, (platform, user_id), default=NotSane) - - other_username = rec.participant - - if self.username == other_username: - # this is a no op - trying to take over itself - return - - - # Make sure we have user confirmation if needed. - # ============================================== - # We need confirmation in whatever combination of the following - # three cases: - # - # - the other participant is not a stub; we are taking the - # account elsewhere away from another viable Gittip - # participant - # - # - the other participant has no other accounts elsewhere; taking - # away the account elsewhere will leave the other Gittip - # participant without any means of logging in, and it will be - # archived and its tips absorbed by us - # - # - we already have an account elsewhere connected from the given - # platform, and it will be handed off to a new stub - # participant - - # other_is_a_real_participant - other_is_a_real_participant = not rec.is_stub - - # this_is_others_last_account_elsewhere - nelsewhere = cursor.one( "SELECT count(*) FROM elsewhere " - "WHERE participant=%s" - , (other_username,) - ) - assert nelsewhere > 0 # sanity check - this_is_others_last_account_elsewhere = (nelsewhere == 1) - - # we_already_have_that_kind_of_account - nparticipants = cursor.one( "SELECT count(*) FROM elsewhere " - "WHERE participant=%s AND platform=%s" - , (self.username, platform) - ) - assert nparticipants in (0, 1) # sanity check - we_already_have_that_kind_of_account = nparticipants == 1 - - need_confirmation = NeedConfirmation( other_is_a_real_participant - , this_is_others_last_account_elsewhere - , we_already_have_that_kind_of_account - ) - if need_confirmation and not have_confirmation: - raise need_confirmation - - - # We have user confirmation. Proceed. - # =================================== - # There is a race condition here. The last person to call this will - # win. XXX: I'm not sure what will happen to the DB and UI for the - # loser. - - - # Move any old account out of the way. - # ==================================== - - if we_already_have_that_kind_of_account: - new_stub_username = reserve_a_random_username(cursor) - cursor.run( "UPDATE elsewhere SET participant=%s " - "WHERE platform=%s AND participant=%s" - , (new_stub_username, platform, self.username) - ) - - - # Do the deal. - # ============ - # If other_is_not_a_stub, then other will have the account - # elsewhere taken away from them with this call. If there are other - # browsing sessions open from that account, they will stay open - # until they expire (XXX Is that okay?) - - cursor.run( "UPDATE elsewhere SET participant=%s " - "WHERE platform=%s AND user_id=%s" - , (self.username, platform, user_id) - ) - - - # Fold the old participant into the new as appropriate. - # ===================================================== - # We want to do this whether or not other is a stub participant. - - if this_is_others_last_account_elsewhere: - - # Take over tips. - # =============== - - x, y = self.username, other_username - cursor.run(CREATE_TEMP_TABLE_FOR_UNIQUE_TIPS) - cursor.run(CONSOLIDATE_TIPS_RECEIVING, dict(live=x, dead=y)) - cursor.run(CONSOLIDATE_TIPS_GIVING, dict(live=x, dead=y)) - cursor.run(ZERO_OUT_OLD_TIPS_RECEIVING, (other_username,)) - cursor.run(ZERO_OUT_OLD_TIPS_GIVING, (other_username,)) - - - # Archive the old participant. - # ============================ - # We always give them a new, random username. We sign out - # the old participant. - - for archive_username in gen_random_usernames(): - try: - username = cursor.one(""" - - UPDATE participants - SET username=%s - , username_lower=%s - , session_token=NULL - , session_expires=now() - WHERE username=%s - RETURNING username - - """, ( archive_username - , archive_username.lower() - , other_username - ), default=NotSane) - except IntegrityError: - continue # archive_username is already taken; - # extremely unlikely, but ... - # XXX But can the UPDATE fail in other ways? - else: - assert username == archive_username - break - - - # Record the absorption. - # ====================== - # This is for preservation of history. - - cursor.run( "INSERT INTO absorptions " - "(absorbed_was, absorbed_by, archived_as) " - "VALUES (%s, %s, %s)" - , ( other_username - , self.username - , archive_username - ) - ) - -# Utter Hack -# ========== - -def utter_hack(db, records): - for rec in records: - yield UtterHack(db, rec) - -class UtterHack(MixinElsewhere): - def __init__(self, db, rec): - self.db = db - for name in rec._fields: - setattr(self, name, getattr(rec, name)) diff --git a/gittip/models/account_elsewhere.py b/gittip/models/account_elsewhere.py new file mode 100644 index 0000000000..a92152174a --- /dev/null +++ b/gittip/models/account_elsewhere.py @@ -0,0 +1,50 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +from postgres.orm import Model + +from gittip.exceptions import ProblemChangingUsername + + +class AccountElsewhere(Model): + + typname = "elsewhere_with_participant" + + def __init__(self, *args, **kwargs): + super(AccountElsewhere, self).__init__(*args, **kwargs) + self.platform_data = getattr(self.platforms, self.platform) + + @property + def html_url(self): + return self.platform_data.account_url.format( + user_id=self.user_id, + user_name=self.user_name, + platform_data=self.platform_data + ) + + def opt_in(self, desired_username): + """Given a desired username, return a User object. + """ + from gittip.security.user import User + self.set_is_locked(False) + user = User.from_username(self.participant.username) + user.sign_in() + assert not user.ANON, self.participant # sanity check + if self.participant.is_claimed: + newly_claimed = False + else: + newly_claimed = True + user.participant.set_as_claimed() + try: + user.participant.change_username(desired_username) + except ProblemChangingUsername: + pass + return user, newly_claimed + + def set_is_locked(self, is_locked): + self.db.run(""" + + UPDATE elsewhere + SET is_locked=%s + WHERE platform=%s AND user_id=%s + + """, (is_locked, self.platform, self.user_id)) diff --git a/gittip/models/community.py b/gittip/models/community.py index f40f2f1dfe..b7115ff4a3 100644 --- a/gittip/models/community.py +++ b/gittip/models/community.py @@ -1,6 +1,5 @@ import re -import gittip from postgres.orm import Model diff --git a/gittip/models/participant.py b/gittip/models/participant.py index f27d6be54b..45ab33de5c 100644 --- a/gittip/models/participant.py +++ b/gittip/models/participant.py @@ -11,15 +11,17 @@ from __future__ import print_function, unicode_literals import datetime -import uuid from decimal import Decimal +import uuid -import gittip -import pytz from aspen import Response from aspen.utils import typecheck -from psycopg2 import IntegrityError from postgres.orm import Model +from psycopg2 import IntegrityError +import pytz + +import gittip +from gittip import NotSane from gittip.exceptions import ( UsernameIsEmpty, UsernameTooLong, @@ -29,11 +31,10 @@ NoSelfTipping, BadAmount, ) - -from gittip.models._mixin_elsewhere import MixinElsewhere from gittip.models._mixin_team import MixinTeam +from gittip.models.account_elsewhere import AccountElsewhere from gittip.utils import canonicalize -from gittip.utils.username import reserve_a_random_username +from gittip.utils.username import gen_random_usernames, reserve_a_random_username ASCII_ALLOWED_IN_USERNAME = set("0123456789" @@ -45,7 +46,7 @@ NANSWERS_THRESHOLD = 0 # configured in wireup.py -class Participant(Model, MixinElsewhere, MixinTeam): +class Participant(Model, MixinTeam): """Represent a Gittip participant. """ @@ -151,6 +152,14 @@ def set_session_expires(self, expires): self.set_attributes(session_expires=session_expires) + # Claimed-ness + # ============ + + @property + def is_claimed(self): + return self.claimed_time is not None + + # Number # ====== @@ -188,23 +197,14 @@ def recreate_api_key(self): def resolve_unclaimed(self): """Given a username, return an URL path. """ - rec = self.db.one( "SELECT platform, user_info " - "FROM elsewhere " - "WHERE participant = %s" + rec = self.db.one( "SELECT platform, user_name " + "FROM elsewhere " + "WHERE participant = %s" , (self.username,) ) if rec is None: - out = None - elif rec.platform == 'bitbucket': - out = '/on/bitbucket/%s/' % rec.user_info['username'] - elif rec.platform == 'github': - out = '/on/github/%s/' % rec.user_info['login'] - elif rec.platform == 'twitter': - out = '/on/twitter/%s/' % rec.user_info['screen_name'] - else: - assert rec.platform == 'openstreetmap' - out = '/on/openstreetmap/%s/' % rec.user_info['username'] - return out + return + return '/on/%s/%s/' % (rec.platform, rec.user_name) def set_as_claimed(self): claimed_time = self.db.one("""\ @@ -273,7 +273,6 @@ def change_username(self, suggested): """ # TODO: reconsider allowing unicode usernames - typecheck(suggested, unicode) suggested = suggested.strip() if not suggested: @@ -540,7 +539,7 @@ def get_giving_for_profile(self): , t.ctime , p.claimed_time , e.platform - , e.user_info + , e.user_name FROM tips t JOIN participants p ON p.username = t.tippee JOIN elsewhere e ON e.participant = t.tippee @@ -551,9 +550,7 @@ def get_giving_for_profile(self): , t.mtime DESC ) AS foo ORDER BY amount DESC - , lower(user_info->'screen_name') - , lower(user_info->'username') - , lower(user_info->'login') + , lower(user_name) """ unclaimed_tips = self.db.all(UNCLAIMED_TIPS, (self.username,)) @@ -687,6 +684,349 @@ def get_age_in_seconds(self): return out + def get_accounts_elsewhere(self): + """Return a dict of AccountElsewhere instances. + """ + accounts = self.db.all(""" + + SELECT elsewhere.*::elsewhere_with_participant + FROM elsewhere + WHERE participant=%s + + """, (self.username,)) + accounts_dict = {account.platform: account for account in accounts} + return accounts_dict + + + def take_over(self, account, have_confirmation=False): + """Given an AccountElsewhere or a tuple (platform_name, user_id), + associate an elsewhere account. + + Returns None or raises NeedConfirmation. + + This method associates an account on another platform (GitHub, Twitter, + etc.) with the given Gittip participant. Every account elsewhere has an + associated Gittip participant account, even if its only a stub + participant (it allows us to track pledges to that account should they + ever decide to join Gittip). + + In certain circumstances, we want to present the user with a + confirmation before proceeding to reconnect the account elsewhere to + the new Gittip account; NeedConfirmation is the signal to request + confirmation. If it was the last account elsewhere connected to the old + Gittip account, then we absorb the old Gittip account into the new one, + effectively archiving the old account. + + Here's what absorbing means: + + - consolidated tips to and fro are set up for the new participant + + Amounts are summed, so if alice tips bob $1 and carl $1, and + then bob absorbs carl, then alice tips bob $2(!) and carl $0. + + And if bob tips alice $1 and carl tips alice $1, and then bob + absorbs carl, then bob tips alice $2(!) and carl tips alice $0. + + The ctime of each new consolidated tip is the older of the two + tips that are being consolidated. + + If alice tips bob $1, and alice absorbs bob, then alice tips + bob $0. + + If alice tips bob $1, and bob absorbs alice, then alice tips + bob $0. + + - all tips to and from the other participant are set to zero + - the absorbed username is released for reuse + - the absorption is recorded in an absorptions table + + This is done in one transaction. + """ + + if isinstance(account, AccountElsewhere): + platform, user_id = account.platform, account.user_id + else: + platform, user_id = account + + CREATE_TEMP_TABLE_FOR_UNIQUE_TIPS = """ + + CREATE TEMP TABLE __temp_unique_tips ON COMMIT drop AS + + -- Get all the latest tips from everyone to everyone. + + SELECT DISTINCT ON (tipper, tippee) + ctime, tipper, tippee, amount + FROM tips + ORDER BY tipper, tippee, mtime DESC; + + """ + + CONSOLIDATE_TIPS_RECEIVING = """ + + -- Create a new set of tips, one for each current tip *to* either + -- the dead or the live account. If a user was tipping both the + -- dead and the live account, then we create one new combined tip + -- to the live account (via the GROUP BY and sum()). + + INSERT INTO tips (ctime, tipper, tippee, amount) + + SELECT min(ctime), tipper, %(live)s AS tippee, sum(amount) + + FROM __temp_unique_tips + + WHERE (tippee = %(dead)s OR tippee = %(live)s) + -- Include tips *to* either the dead or live account. + + AND NOT (tipper = %(dead)s OR tipper = %(live)s) + -- Don't include tips *from* the dead or live account, + -- lest we convert cross-tipping to self-tipping. + + AND amount > 0 + -- Don't include zeroed out tips, so we avoid a no-op + -- zero tip entry. + + GROUP BY tipper + + """ + + CONSOLIDATE_TIPS_GIVING = """ + + -- Create a new set of tips, one for each current tip *from* either + -- the dead or the live account. If both the dead and the live + -- account were tipping a given user, then we create one new + -- combined tip from the live account (via the GROUP BY and sum()). + + INSERT INTO tips (ctime, tipper, tippee, amount) + + SELECT min(ctime), %(live)s AS tipper, tippee, sum(amount) + + FROM __temp_unique_tips + + WHERE (tipper = %(dead)s OR tipper = %(live)s) + -- Include tips *from* either the dead or live account. + + AND NOT (tippee = %(dead)s OR tippee = %(live)s) + -- Don't include tips *to* the dead or live account, + -- lest we convert cross-tipping to self-tipping. + + AND amount > 0 + -- Don't include zeroed out tips, so we avoid a no-op + -- zero tip entry. + + GROUP BY tippee + + """ + + ZERO_OUT_OLD_TIPS_RECEIVING = """ + + INSERT INTO tips (ctime, tipper, tippee, amount) + + SELECT ctime, tipper, tippee, 0 AS amount + FROM __temp_unique_tips + WHERE tippee=%s AND amount > 0 + + """ + + ZERO_OUT_OLD_TIPS_GIVING = """ + + INSERT INTO tips (ctime, tipper, tippee, amount) + + SELECT ctime, tipper, tippee, 0 AS amount + FROM __temp_unique_tips + WHERE tipper=%s AND amount > 0 + + """ + + with self.db.get_cursor() as cursor: + + # Load the existing connection. + # ============================= + # Every account elsewhere has at least a stub participant account + # on Gittip. + + rec = cursor.one(""" + + SELECT participant + , claimed_time IS NULL AS is_stub + FROM elsewhere + JOIN participants ON participant=participants.username + WHERE elsewhere.platform=%s AND elsewhere.user_id=%s + + """, (platform, user_id), default=NotSane) + + other_username = rec.participant + + if self.username == other_username: + # this is a no op - trying to take over itself + return + + + # Make sure we have user confirmation if needed. + # ============================================== + # We need confirmation in whatever combination of the following + # three cases: + # + # - the other participant is not a stub; we are taking the + # account elsewhere away from another viable Gittip + # participant + # + # - the other participant has no other accounts elsewhere; taking + # away the account elsewhere will leave the other Gittip + # participant without any means of logging in, and it will be + # archived and its tips absorbed by us + # + # - we already have an account elsewhere connected from the given + # platform, and it will be handed off to a new stub + # participant + + # other_is_a_real_participant + other_is_a_real_participant = not rec.is_stub + + # this_is_others_last_account_elsewhere + nelsewhere = cursor.one( "SELECT count(*) FROM elsewhere " + "WHERE participant=%s" + , (other_username,) + ) + assert nelsewhere > 0 # sanity check + this_is_others_last_account_elsewhere = (nelsewhere == 1) + + # we_already_have_that_kind_of_account + nparticipants = cursor.one( "SELECT count(*) FROM elsewhere " + "WHERE participant=%s AND platform=%s" + , (self.username, platform) + ) + assert nparticipants in (0, 1) # sanity check + we_already_have_that_kind_of_account = nparticipants == 1 + + need_confirmation = NeedConfirmation( other_is_a_real_participant + , this_is_others_last_account_elsewhere + , we_already_have_that_kind_of_account + ) + if need_confirmation and not have_confirmation: + raise need_confirmation + + + # We have user confirmation. Proceed. + # =================================== + # There is a race condition here. The last person to call this will + # win. XXX: I'm not sure what will happen to the DB and UI for the + # loser. + + + # Move any old account out of the way. + # ==================================== + + if we_already_have_that_kind_of_account: + new_stub_username = reserve_a_random_username(cursor) + cursor.run( "UPDATE elsewhere SET participant=%s " + "WHERE platform=%s AND participant=%s" + , (new_stub_username, platform, self.username) + ) + + + # Do the deal. + # ============ + # If other_is_not_a_stub, then other will have the account + # elsewhere taken away from them with this call. If there are other + # browsing sessions open from that account, they will stay open + # until they expire (XXX Is that okay?) + + cursor.run( "UPDATE elsewhere SET participant=%s " + "WHERE platform=%s AND user_id=%s" + , (self.username, platform, user_id) + ) + + + # Fold the old participant into the new as appropriate. + # ===================================================== + # We want to do this whether or not other is a stub participant. + + if this_is_others_last_account_elsewhere: + + # Take over tips. + # =============== + + x, y = self.username, other_username + cursor.run(CREATE_TEMP_TABLE_FOR_UNIQUE_TIPS) + cursor.run(CONSOLIDATE_TIPS_RECEIVING, dict(live=x, dead=y)) + cursor.run(CONSOLIDATE_TIPS_GIVING, dict(live=x, dead=y)) + cursor.run(ZERO_OUT_OLD_TIPS_RECEIVING, (other_username,)) + cursor.run(ZERO_OUT_OLD_TIPS_GIVING, (other_username,)) + + + # Archive the old participant. + # ============================ + # We always give them a new, random username. We sign out + # the old participant. + + for archive_username in gen_random_usernames(): + try: + username = cursor.one(""" + + UPDATE participants + SET username=%s + , username_lower=%s + , session_token=NULL + , session_expires=now() + WHERE username=%s + RETURNING username + + """, ( archive_username + , archive_username.lower() + , other_username + ), default=NotSane) + except IntegrityError: + continue # archive_username is already taken; + # extremely unlikely, but ... + # XXX But can the UPDATE fail in other ways? + else: + assert username == archive_username + break + + + # Record the absorption. + # ====================== + # This is for preservation of history. + + cursor.run( "INSERT INTO absorptions " + "(absorbed_was, absorbed_by, archived_as) " + "VALUES (%s, %s, %s)" + , ( other_username + , self.username + , archive_username + ) + ) + + +class NeedConfirmation(Exception): + """Represent the case where we need user confirmation during a merge. + + This is used in the workflow for merging one participant into another. + + """ + + def __init__(self, a, b, c): + self.other_is_a_real_participant = a + self.this_is_others_last_account_elsewhere = b + self.we_already_have_that_kind_of_account = c + self._all = (a, b, c) + + def __repr__(self): + return "" % self._all + __str__ = __repr__ + + def __eq__(self, other): + return self._all == other._all + + def __ne__(self, other): + return not self.__eq__(other) + + def __nonzero__(self): + # bool(need_confirmation) + A, B, C = self._all + return A or C + + def typecast(request): """Given a Request, raise Response or return Participant. diff --git a/gittip/security/crypto.py b/gittip/security/crypto.py index a7449babc0..85cb218424 100644 --- a/gittip/security/crypto.py +++ b/gittip/security/crypto.py @@ -83,7 +83,7 @@ def salted_hmac(key_salt, value, secret=None): """ if secret is None: raise NotImplementedError - secret = settings.SECRET_KEY + #secret = settings.SECRET_KEY # We need to generate a derived key from our base key. We can do this by # passing the key_salt and our base key through a pseudo-random function and diff --git a/gittip/testing/__init__.py b/gittip/testing/__init__.py index 815623c14b..bc9a1e7b5c 100644 --- a/gittip/testing/__init__.py +++ b/gittip/testing/__init__.py @@ -12,6 +12,7 @@ from aspen import resources from aspen.testing.client import Client from gittip.billing.payday import Payday +from gittip.elsewhere import UserInfo from gittip.models.participant import Participant from gittip.security.user import User from gittip import wireup @@ -25,53 +26,6 @@ FIXTURES_ROOT = join(TOP, 'tests', 'fixtures') -DUMMY_GITHUB_JSON = u'{"html_url":"https://github.com/whit537","type":"User",'\ -'"public_repos":25,"blog":"http://whit537.org/","gravatar_id":"fb054b407a6461'\ -'e417ee6b6ae084da37","public_gists":29,"following":15,"updated_at":"2013-01-1'\ -'4T13:43:23Z","company":"Gittip","events_url":"https://api.github.com/users/w'\ -'hit537/events{/privacy}","repos_url":"https://api.github.com/users/whit537/r'\ -'epos","gists_url":"https://api.github.com/users/whit537/gists{/gist_id}","em'\ -'ail":"chad@zetaweb.com","organizations_url":"https://api.github.com/users/wh'\ -'it537/orgs","hireable":false,"received_events_url":"https://api.github.com/u'\ -'sers/whit537/received_events","starred_url":"https://api.github.com/users/wh'\ -'it537/starred{/owner}{/repo}","login":"whit537","created_at":"2009-10-03T02:'\ -'47:57Z","bio":"","url":"https://api.github.com/users/whit537","avatar_url":"'\ -'https://secure.gravatar.com/avatar/fb054b407a6461e417ee6b6ae084da37?d=https:'\ -'//a248.e.akamai.net/assets.github.com%2Fimages%2Fgravatars%2Fgravatar-user-4'\ -'20.png","followers":90,"name":"Chad Whitacre","followers_url":"https://api.g'\ -'ithub.com/users/whit537/followers","following_url":"https://api.github.com/u'\ -'sers/whit537/following","id":134455,"location":"Pittsburgh, PA","subscriptio'\ -'ns_url":"https://api.github.com/users/whit537/subscriptions"}' -# JSON data as returned from github for whit537 ;) - -GITHUB_USER_UNREGISTERED_LGTEST = u'{"public_repos":0,"html_url":"https://git'\ -'hub.com/lgtest","type":"User","repos_url":"https://api.github.com/users/lgte'\ -'st/repos","gravatar_id":"d41d8cd98f00b204e9800998ecf8427e","following":0,"pu'\ -'blic_gists":0,"updated_at":"2013-01-04T17:24:57Z","received_events_url":"htt'\ -'ps://api.github.com/users/lgtest/received_events","gists_url":"https://api.g'\ -'ithub.com/users/lgtest/gists{/gist_id}","events_url":"https://api.github.com'\ -'/users/lgtest/events{/privacy}","organizations_url":"https://api.github.com/'\ -'users/lgtest/orgs","avatar_url":"https://secure.gravatar.com/avatar/d41d8cd9'\ -'8f00b204e9800998ecf8427e?d=https://a248.e.akamai.net/assets.github.com%2Fima'\ -'ges%2Fgravatars%2Fgravatar-user-420.png","login":"lgtest","created_at":"2012'\ -'-05-24T20:09:07Z","starred_url":"https://api.github.com/users/lgtest/starred'\ -'{/owner}{/repo}","url":"https://api.github.com/users/lgtest","followers":0,"'\ -'followers_url":"https://api.github.com/users/lgtest/followers","following_ur'\ -'l":"https://api.github.com/users/lgtest/following","id":1775515,"subscriptio'\ -'ns_url":"https://api.github.com/users/lgtest/subscriptions"}' -# JSON data as returned from github for unregistered user ``lgtest`` - -DUMMY_BOUNTYSOURCE_JSON = u'{"slug": "6-corytheboyd","updated_at": "2013-05-2'\ -'4T01:45:20Z","last_name": "Boyd","id": 6,"last_seen_at": "2013-05-24T01:45:2'\ -'0Z","email": "corytheboyd@gmail.com","fundraisers": [],"frontend_path": "#us'\ -'ers/6-corytheboyd","display_name": "corytheboyd","frontend_url": "https://ww'\ -'w.bountysource.com/#users/6-corytheboyd","created_at": "2012-09-14T03:28:07Z'\ -'","first_name": "Cory","bounties": [],"image_url": "https://secure.gravatar.'\ -'com/avatar/bdeaea505d059ccf23d8de5714ae7f73?d=https://a248.e.akamai.net/asse'\ -'ts.github.com%2Fimages%2Fgravatars%2Fgravatar-user-420.png"}' -# JSON data as returned from bountysource for corytheboyd! hello, whit537 ;) - - class ClientWithAuth(Client): def __init__(self, *a, **kw): @@ -105,6 +59,7 @@ class Harness(unittest.TestCase): def setUpClass(cls): cls.client = ClientWithAuth(www_root=WWW_ROOT, project_root=PROJECT_ROOT) cls.db = cls.client.website.db + cls.platforms = cls.client.website.platforms cls.tablenames = cls.db.all("SELECT tablename FROM pg_tables " "WHERE schemaname='public'") cls.seq = 0 @@ -156,6 +111,12 @@ def clear_tables(self): tablenames.insert(0, tablename) + def make_elsewhere(self, platform, user_id, user_name, **kw): + platform = getattr(self.platforms, platform) + info = UserInfo(user_id=unicode(user_id), user_name=user_name, **kw) + return platform.upsert(info) + + def show_table(self, table): print('\n{:=^80}'.format(table)) data = self.db.all('select * from '+table, back_as='namedtuple') @@ -180,17 +141,16 @@ def make_participant(self, username, **kw): participant = Participant.with_random_username() participant.change_username(username) - return self.update_participant(participant, **kw) - - def update_participant(self, participant, **kw): if 'elsewhere' in kw or 'claimed_time' in kw: username = participant.username platform = kw.pop('elsewhere', 'github') - user_info = dict(login=username) self.seq += 1 - self.db.run("INSERT INTO elsewhere (platform, user_id, participant, user_info) " - "VALUES (%s,%s,%s,%s)", (platform, self.seq, username, user_info)) + self.db.run(""" + INSERT INTO elsewhere + (platform, user_id, user_name, participant) + VALUES (%s,%s,%s,%s) + """, (platform, self.seq, username, username)) # brute force update for use in testing for k,v in kw.items(): diff --git a/gittip/testing/elsewhere.py b/gittip/testing/elsewhere.py new file mode 100644 index 0000000000..96dc12d772 --- /dev/null +++ b/gittip/testing/elsewhere.py @@ -0,0 +1,285 @@ +# -*- coding: utf-8 -*- + +""" +Examples of data returned by the APIs of the elsewhere platforms. + +They are wrapped in lambdas to prevent tests from persistently modifying the +data. +""" + +import xml.etree.ElementTree as ET + +bitbucket = lambda: { + "repositories": [ + { + "scm": "hg", + "has_wiki": True, + "last_updated": "2012-03-16T23:36:38.019", + "no_forks": None, + "created_on": "2012-03-16T23:34:46.740", + "owner": "whit537", + "logo": "https://d3oaxc4q5k2d6q.cloudfront.net/m/6fac1fb24100/img/language-avatars/default_16.png", + "email_mailinglist": "", + "is_mq": False, + "size": 142818, + "read_only": False, + "fork_of": { + "scm": "hg", + "has_wiki": True, + "last_updated": "2014-02-01T03:41:46.920", + "no_forks": None, + "created_on": "2010-07-17T16:12:34.381", + "owner": "jaraco", + "logo": "https://d3oaxc4q5k2d6q.cloudfront.net/m/6fac1fb24100/img/language-avatars/python_16.png", + "email_mailinglist": "", + "is_mq": False, + "size": 316601, + "read_only": False, + "creator": None, + "state": "available", + "utc_created_on": "2010-07-17 14:12:34+00:00", + "website": "", + "description": "Inspried by jezdez.setuptools_hg, and building on that work, hgtools provides tools for developing with mercurial and setuptools/distribute (specifically a file-finder plugin and automatic repo tag versioning).\r\n\r\nThe underlying library is designed to be extensible for other applications to build other functionality that depends on mercurial, whether using the 'hg' command or the mercurial libraries directly.", + "has_issues": True, + "is_fork": True, + "slug": "hgtools", + "is_private": False, + "name": "hgtools", + "language": "python", + "utc_last_updated": "2014-02-01 02:41:46+00:00", + "email_writers": True, + "no_public_forks": False, + "resource_uri": "/1.0/repositories/jaraco/hgtools" + }, + "mq_of": { + "scm": "hg", + "has_wiki": True, + "last_updated": "2014-02-01T03:41:46.920", + "no_forks": None, + "created_on": "2010-07-17T16:12:34.381", + "owner": "jaraco", + "logo": "https://d3oaxc4q5k2d6q.cloudfront.net/m/6fac1fb24100/img/language-avatars/python_16.png", + "email_mailinglist": "", + "is_mq": False, + "size": 316601, + "read_only": False, + "creator": None, + "state": "available", + "utc_created_on": "2010-07-17 14:12:34+00:00", + "website": "", + "description": "Inspried by jezdez.setuptools_hg, and building on that work, hgtools provides tools for developing with mercurial and setuptools/distribute (specifically a file-finder plugin and automatic repo tag versioning).\r\n\r\nThe underlying library is designed to be extensible for other applications to build other functionality that depends on mercurial, whether using the 'hg' command or the mercurial libraries directly.", + "has_issues": True, + "is_fork": True, + "slug": "hgtools", + "is_private": False, + "name": "hgtools", + "language": "python", + "utc_last_updated": "2014-02-01 02:41:46+00:00", + "email_writers": True, + "no_public_forks": False, + "resource_uri": "/1.0/repositories/jaraco/hgtools" + }, + "state": "available", + "utc_created_on": "2012-03-16 22:34:46+00:00", + "website": None, + "description": "I'm forking to fix another bug case in issue #7.", + "has_issues": True, + "is_fork": True, + "slug": "hgtools", + "is_private": False, + "name": "hgtools", + "language": "", + "utc_last_updated": "2012-03-16 22:36:38+00:00", + "email_writers": True, + "no_public_forks": False, + "creator": None, + "resource_uri": "/1.0/repositories/whit537/hgtools" + } + ], + "user": { + "username": "whit537", + "first_name": "Chad", + "last_name": "Whitacre", + "display_name": "Chad Whitacre", + "is_team": False, + "avatar": "https://secure.gravatar.com/avatar/5698bc43665106a28833ef61c8a9f67f?d=https%3A%2F%2Fd3oaxc4q5k2d6q.cloudfront.net%2Fm%2F6fac1fb24100%2Fimg%2Fdefault_avatar%2F32%2Fuser_blue.png&s=32", + "resource_uri": "/1.0/users/whit537" + } +} + +bountysource = lambda: { + "bio": "Code alchemist at Bountysource.", + "twitter_account": { + "uid": 313084547, + "followers": None, + "following": None, + "image_url": "https://cloudinary-a.akamaihd.net/bountysource/image/twitter_name/d_noaoqqwxegvmulwus0un.png,c_pad,w_100,h_100/corytheboyd.png", + "login": "corytheboyd", + "id": 2105 + }, + "display_name": "corytheboyd", + "url": "", + "type": "Person", + "created_at": "2012-09-14T03:28:07Z", + "slug": "6-corytheboyd", + "facebook_account": { + "uid": 589244295, + "followers": 0, + "following": 0, + "image_url": "https://cloudinary-a.akamaihd.net/bountysource/image/facebook/d_noaoqqwxegvmulwus0un.png,c_pad,w_100,h_100/corytheboyd.jpg", + "login": "corytheboyd", + "id": 2103 + }, + "gittip_account": { + "uid": 17306, + "followers": 0, + "following": 0, + "image_url": "https://cloudinary-a.akamaihd.net/bountysource/image/gravatar/d_noaoqqwxegvmulwus0un.png,c_pad,w_100,h_100/bdeaea505d059ccf23d8de5714ae7f73", + "login": "corytheboyd", + "id": 2067 + }, + "large_image_url": "https://cloudinary-a.akamaihd.net/bountysource/image/twitter_name/d_noaoqqwxegvmulwus0un.png,c_pad,w_400,h_400/corytheboyd.png", + "frontend_path": "/users/6-corytheboyd", + "image_url": "https://cloudinary-a.akamaihd.net/bountysource/image/twitter_name/d_noaoqqwxegvmulwus0un.png,c_pad,w_100,h_100/corytheboyd.png", + "location": "San Francisco, CA", + "medium_image_url": "https://cloudinary-a.akamaihd.net/bountysource/image/twitter_name/d_noaoqqwxegvmulwus0un.png,c_pad,w_200,h_200/corytheboyd.png", + "frontend_url": "https://www.bountysource.com/users/6-corytheboyd", + "github_account": { + "uid": 692632, + "followers": 11, + "following": 4, + "image_url": "https://cloudinary-a.akamaihd.net/bountysource/image/gravatar/d_noaoqqwxegvmulwus0un.png,c_pad,w_100,h_100/bdeaea505d059ccf23d8de5714ae7f73", + "login": "corytheboyd", + "id": 89, + "permissions": [ + "public_repo" + ] + }, + "company": "Bountysource", + "id": 6, + "public_email": "cory@bountysource.com" +} + +github = lambda: { + "bio": "", + "updated_at": "2013-01-14T13:43:23Z", + "gravatar_id": "fb054b407a6461e417ee6b6ae084da37", + "hireable": False, + "id": 134455, + "followers_url": "https://api.github.com/users/whit537/followers", + "following_url": "https://api.github.com/users/whit537/following", + "blog": "http://whit537.org/", + "followers": 90, + "location": "Pittsburgh, PA", + "type": "User", + "email": "chad@zetaweb.com", + "public_repos": 25, + "events_url": "https://api.github.com/users/whit537/events{/privacy}", + "company": "Gittip", + "gists_url": "https://api.github.com/users/whit537/gists{/gist_id}", + "html_url": "https://github.com/whit537", + "subscriptions_url": "https://api.github.com/users/whit537/subscriptions", + "received_events_url": "https://api.github.com/users/whit537/received_events", + "starred_url": "https://api.github.com/users/whit537/starred{/owner}{/repo}", + "public_gists": 29, + "name": "Chad Whitacre", + "organizations_url": "https://api.github.com/users/whit537/orgs", + "url": "https://api.github.com/users/whit537", + "created_at": "2009-10-03T02:47:57Z", + "avatar_url": "https://secure.gravatar.com/avatar/fb054b407a6461e417ee6b6ae084da37?d=https://a248.e.akamai.net/assets.github.com%2Fimages%2Fgravatars%2Fgravatar-user-420.png", + "repos_url": "https://api.github.com/users/whit537/repos", + "following": 15, + "login": "whit537" +} + +openstreetmap = lambda: ET.fromstring(""" + + + + + + + + + + + + + + + +""") + +twitter = lambda: { + "lang": "en", + "utc_offset": 3600, + "statuses_count": 1339, + "follow_request_sent": None, + "friends_count": 81, + "profile_use_background_image": True, + "contributors_enabled": False, + "profile_link_color": "0084B4", + "profile_image_url": "http://pbs.twimg.com/profile_images/3502698593/36a503f65df33aea1a59faea77a57e73_normal.png", + "time_zone": "Paris", + "notifications": None, + "is_translator": False, + "favourites_count": 81, + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_color": "C0DEED", + "id": 23608307, + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "description": "#Freelance computer programmer from France. In English: #FreeSoftware and #BasicIncome. In French: #LogicielLibre, #RevenuDeBase and #Démocratie/#TirageAuSort.", + "is_translation_enabled": False, + "default_profile": True, + "profile_background_tile": False, + "verified": False, + "screen_name": "Changaco", + "entities": { + "url": { + "urls": [ + { + "url": "http://t.co/2VUhacI9SG", + "indices": [ + 0, + 22 + ], + "expanded_url": "http://changaco.oy.lc/", + "display_url": "changaco.oy.lc" + } + ] + }, + "description": { + "urls": [] + } + }, + "url": "http://t.co/2VUhacI9SG", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/3502698593/36a503f65df33aea1a59faea77a57e73_normal.png", + "profile_sidebar_fill_color": "DDEEF6", + "location": "France", + "name": "Changaco", + "geo_enabled": False, + "profile_text_color": "333333", + "followers_count": 94, + "profile_sidebar_border_color": "C0DEED", + "id_str": "23608307", + "default_profile_image": False, + "following": None, + "protected": False, + "created_at": "Tue Mar 10 15:58:07 +0000 2009", + "listed_count": 7 +} + +venmo = lambda: { + "about": "No Short Bio", + "date_joined": "2013-09-11T19:57:53", + "display_name": "Thomas Boyt", + "email": None, + "first_name": "Thomas", + "friends_count": 30, + "id": "1242868517699584789", + "is_friend": False, + "last_name": "Boyt", + "phone": None, + "profile_picture_url": "https://s3.amazonaws.com/venmo/no-image.gif", + "username": "thomas-boyt" +} diff --git a/gittip/utils/__init__.py b/gittip/utils/__init__.py index 025f4a3044..260ad1c7ba 100644 --- a/gittip/utils/__init__.py +++ b/gittip/utils/__init__.py @@ -1,7 +1,6 @@ import locale import time -import gittip import re from aspen import log_dammit, Response from aspen.utils import typecheck @@ -365,8 +364,8 @@ def update_homepage_queries_once(db): cursor.execute("DELETE FROM homepage_top_givers") cursor.execute(""" - INSERT INTO homepage_top_givers (username, anonymous, amount) - SELECT tipper, anonymous_giving, sum(amount) AS amount + INSERT INTO homepage_top_givers (username, anonymous, amount, avatar_url) + SELECT tipper, anonymous_giving, sum(amount) AS amount, avatar_url FROM ( SELECT DISTINCT ON (tipper, tippee) amount , tipper @@ -382,34 +381,17 @@ def update_homepage_queries_once(db): ) AS foo JOIN participants p ON p.username = tipper WHERE is_suspicious IS NOT true - GROUP BY tipper, anonymous_giving - ORDER BY amount DESC; + GROUP BY tipper, anonymous_giving, avatar_url + ORDER BY amount DESC + LIMIT 100; """.strip()) - cursor.execute(""" - - UPDATE homepage_top_givers - SET gravatar_id = ( SELECT user_info->'gravatar_id' - FROM elsewhere - WHERE participant=username - AND platform='github' - ) - """) - cursor.execute(""" - - UPDATE homepage_top_givers - SET twitter_pic = ( SELECT user_info->'profile_image_url_https' - FROM elsewhere - WHERE participant=username - AND platform='twitter' - ) - """) cursor.execute("DELETE FROM homepage_top_receivers") cursor.execute(""" - INSERT INTO homepage_top_receivers (username, anonymous, amount, claimed_time) - SELECT tippee, anonymous_receiving, sum(amount) AS amount, claimed_time + INSERT INTO homepage_top_receivers (username, anonymous, amount, avatar_url) + SELECT tippee, anonymous_receiving, sum(amount) AS amount, avatar_url FROM ( SELECT DISTINCT ON (tipper, tippee) amount , tippee @@ -424,28 +406,12 @@ def update_homepage_queries_once(db): ) AS foo JOIN participants p ON p.username = tippee WHERE is_suspicious IS NOT true - GROUP BY tippee, anonymous_receiving, claimed_time - ORDER BY amount DESC; + GROUP BY tippee, anonymous_receiving, avatar_url + ORDER BY amount DESC + LIMIT 100; """.strip()) - cursor.execute(""" - - UPDATE homepage_top_receivers - SET gravatar_id = ( SELECT user_info->'gravatar_id' - FROM elsewhere - WHERE participant=username - AND platform='github' - ) - """) - cursor.execute(""" - UPDATE homepage_top_receivers - SET twitter_pic = ( SELECT user_info->'profile_image_url_https' - FROM elsewhere - WHERE participant=username - AND platform='twitter' - ) - """) end = time.time() elapsed = end - start log_dammit("updated homepage queries in %.2f seconds" % elapsed) @@ -466,10 +432,8 @@ def wrapper(*a, **kw): return ret return wrapper -def redirect_confirmation(website, request): - from aspen import resources - request.internally_redirected_from = request.fs - request.fs = website.www_root + '/on/confirm.html.spt' - request.resource = resources.get(request) - raise request.resource.respond(request) +def get_avatar_url(obj): + if not obj.avatar_url: + return '/assets/-/avatar-default.gif' + return obj.avatar_url diff --git a/gittip/utils/fake_data.py b/gittip/utils/fake_data.py index a387b66324..ab467b98f0 100644 --- a/gittip/utils/fake_data.py +++ b/gittip/utils/fake_data.py @@ -1,17 +1,16 @@ from faker import Factory from gittip import wireup, MAX_TIP, MIN_TIP +from gittip.elsewhere import PLATFORMS from gittip.models.participant import Participant +import datetime import decimal import random import string -import datetime faker = Factory.create() -platforms = ['github', 'twitter', 'bitbucket', 'openstreetmap'] - def _fake_thing(db, tablename, **kw): column_names = [] @@ -61,7 +60,7 @@ def fake_participant(db, number="singular", is_admin=False): """Create a fake User. """ username = faker.first_name() + fake_text_id(3) - d = _fake_thing( db + _fake_thing( db , "participants" , id=fake_int_id() , username=username @@ -108,46 +107,21 @@ def fake_tip(db, tipper, tippee): ) -def fake_elsewhere(db, participant, platform=None): +def fake_elsewhere(db, participant, platform): """Create a fake elsewhere. """ - if platform is None: - platform = random.choice(platforms) - - info_templates = { - "github": { - "name": participant.username, - "html_url": "https://github.com/" + participant.username, - "type": "User", - "login": participant.username - }, - "twitter": { - "name": participant.username, - "html_url": "https://twitter.com/" + participant.username, - "screen_name": participant.username - }, - "bitbucket": { - "display_name": participant.username, - "username": participant.username, - "is_team": "False", - "html_url": "https://bitbucket.org/" + participant.username, - }, - "openstreetmap": { - "username": participant.username, - "html_url": "https://openstreetmap/user/" + participant.username, - } - } - _fake_thing( db , "elsewhere" , id=fake_int_id() , platform=platform , user_id=fake_text_id() + , user_name=participant.username , is_locked=False , participant=participant.username - , user_info=info_templates[platform] + , extra_info=None ) + def fake_transfer(db, tipper, tippee): return _fake_thing( db , "transfers" @@ -171,7 +145,7 @@ def populate_db(db, num_participants=100, num_tips=200, num_teams=5, num_transfe for p in participants: #All participants get between 1 and 3 elsewheres num_elsewheres = random.randint(1, 3) - for platform_name in platforms[:num_elsewheres]: + for platform_name in random.sample(PLATFORMS, num_elsewheres): fake_elsewhere(db, p, platform_name) #Make teams diff --git a/gittip/wireup.py b/gittip/wireup.py index 5cd99c4ce7..f4ae489e93 100644 --- a/gittip/wireup.py +++ b/gittip/wireup.py @@ -8,8 +8,15 @@ import balanced import gittip import raven -import psycopg2 import stripe +from gittip.elsewhere import PlatformRegistry +from gittip.elsewhere.bitbucket import Bitbucket +from gittip.elsewhere.bountysource import Bountysource +from gittip.elsewhere.github import GitHub +from gittip.elsewhere.openstreetmap import OpenStreetMap +from gittip.elsewhere.twitter import Twitter +from gittip.elsewhere.venmo import Venmo +from gittip.models.account_elsewhere import AccountElsewhere from gittip.models.community import Community from gittip.models.participant import Participant from gittip.models import GittipDB @@ -25,11 +32,8 @@ def db(): maxconn = int(os.environ['DATABASE_MAXCONN']) db = GittipDB(dburl, maxconn=maxconn) - # register hstore type - with db.get_cursor() as cursor: - psycopg2.extras.register_hstore(cursor, globally=True, unicode=True) - db.register_model(Community) + db.register_model(AccountElsewhere) db.register_model(Participant) return db @@ -141,6 +145,7 @@ def nanswers(): class BadEnvironment(SystemExit): pass + def envvars(website): missing_keys = [] @@ -163,33 +168,53 @@ def envvar(key, cast=None): def is_yesish(val): return val.lower() in ('1', 'true', 'yes') - website.bitbucket_consumer_key = envvar('BITBUCKET_CONSUMER_KEY') - website.bitbucket_consumer_secret = envvar('BITBUCKET_CONSUMER_SECRET') - website.bitbucket_callback = envvar('BITBUCKET_CALLBACK') - - website.github_client_id = envvar('GITHUB_CLIENT_ID') - website.github_client_secret = envvar('GITHUB_CLIENT_SECRET') - website.github_callback = envvar('GITHUB_CALLBACK') - - website.twitter_consumer_key = envvar('TWITTER_CONSUMER_KEY') - website.twitter_consumer_secret = envvar('TWITTER_CONSUMER_SECRET') - website.twitter_access_token = envvar('TWITTER_ACCESS_TOKEN') - website.twitter_access_token_secret = envvar('TWITTER_ACCESS_TOKEN_SECRET') - website.twitter_callback = envvar('TWITTER_CALLBACK') - - website.bountysource_www_host = envvar('BOUNTYSOURCE_WWW_HOST') - website.bountysource_api_host = envvar('BOUNTYSOURCE_API_HOST') - website.bountysource_api_secret = envvar('BOUNTYSOURCE_API_SECRET') - website.bountysource_callback = envvar('BOUNTYSOURCE_CALLBACK') - - website.venmo_client_id = envvar('VENMO_CLIENT_ID') - website.venmo_client_secret = envvar('VENMO_CLIENT_SECRET') - website.venmo_callback = envvar('VENMO_CALLBACK') - - website.openstreetmap_api = envvar('OPENSTREETMAP_API') - website.openstreetmap_consumer_key = envvar('OPENSTREETMAP_CONSUMER_KEY') - website.openstreetmap_consumer_secret = envvar('OPENSTREETMAP_CONSUMER_SECRET') - website.openstreetmap_callback = envvar('OPENSTREETMAP_CALLBACK') + signin_platforms = [ + Twitter( + website.db, + envvar('TWITTER_CONSUMER_KEY'), + envvar('TWITTER_CONSUMER_SECRET'), + envvar('TWITTER_CALLBACK'), + ), + GitHub( + website.db, + envvar('GITHUB_CLIENT_ID'), + envvar('GITHUB_CLIENT_SECRET'), + envvar('GITHUB_CALLBACK'), + ), + Bitbucket( + website.db, + envvar('BITBUCKET_CONSUMER_KEY'), + envvar('BITBUCKET_CONSUMER_SECRET'), + envvar('BITBUCKET_CALLBACK'), + ), + OpenStreetMap( + website.db, + envvar('OPENSTREETMAP_CONSUMER_KEY'), + envvar('OPENSTREETMAP_CONSUMER_SECRET'), + envvar('OPENSTREETMAP_CALLBACK'), + envvar('OPENSTREETMAP_API_URL'), + envvar('OPENSTREETMAP_AUTH_URL'), + ), + ] + website.signin_platforms = PlatformRegistry(signin_platforms) + other_platforms = [ + Bountysource( + website.db, + None, + envvar('BOUNTYSOURCE_API_SECRET'), + envvar('BOUNTYSOURCE_CALLBACK'), + envvar('BOUNTYSOURCE_API_HOST'), + envvar('BOUNTYSOURCE_WWW_HOST'), + ), + Venmo( + website.db, + envvar('VENMO_CLIENT_ID'), + envvar('VENMO_CLIENT_SECRET'), + envvar('VENMO_CALLBACK'), + ), + ] + platforms = signin_platforms + other_platforms + website.platforms = AccountElsewhere.platforms = PlatformRegistry(platforms) website.asset_version_url = envvar('GITTIP_ASSET_VERSION_URL') \ .replace('%version', website.version) diff --git a/requirements.txt b/requirements.txt index b0810a3deb..31bd8db83f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,8 @@ ./vendor/chardet-1.0.1.tar.gz ./vendor/oauthlib-0.4.2.tar.gz ./vendor/requests-1.2.3.tar.gz -./vendor/requests-oauthlib-0.3.2.tar.gz +./vendor/requests-oauthlib-0.4.0.tar.gz +./vendor/xmltodict-0.8.4.tar.gz ./vendor/stripe-1.9.1.tar.gz diff --git a/scss/layout.scss b/scss/layout.scss index 06bbc63e4d..cce9865cc4 100644 --- a/scss/layout.scss +++ b/scss/layout.scss @@ -514,7 +514,7 @@ text-transform: uppercase; } - h2 { + h2, #members th { color: $brown; font: bold 14px $Helvetica; letter-spacing: 1px; diff --git a/scss/lib/_dropdown.scss b/scss/lib/_dropdown.scss index 65857f00a1..454d6144cc 100644 --- a/scss/lib/_dropdown.scss +++ b/scss/lib/_dropdown.scss @@ -21,7 +21,7 @@ .open .dropdown-toggle { outline: 0; } -a.dropdown-toggle{ +button.dropdown-toggle{ width: 75px; text-align: center; } @@ -68,7 +68,7 @@ a.dropdown-toggle{ background-color: #e5e5e5; border-bottom: 1px solid #ffffff; } -.dropdown-menu > li > a { +.dropdown-menu > li button { display: block; padding: 3px 20px; clear: both; @@ -77,10 +77,10 @@ a.dropdown-toggle{ color: #333333; white-space: nowrap; } -.dropdown-menu > li > a:hover, -.dropdown-menu > li > a:focus, -.dropdown-submenu:hover > a, -.dropdown-submenu:focus > a { +.dropdown-menu > li button:hover, +.dropdown-menu > li button:focus, +.dropdown-submenu:hover button, +.dropdown-submenu:focus button { text-decoration: none; color: #ffffff; background-color: #0081c2; @@ -92,9 +92,9 @@ a.dropdown-toggle{ background-repeat: repeat-x; filter: unquote("progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0077b3', GradientType=0)"); } -.dropdown-menu > .active > a, -.dropdown-menu > .active > a:hover, -.dropdown-menu > .active > a:focus { +.dropdown-menu > .active button, +.dropdown-menu > .active button:hover, +.dropdown-menu > .active button:focus { color: #ffffff; text-decoration: none; outline: 0; @@ -107,13 +107,13 @@ a.dropdown-toggle{ background-repeat: repeat-x; filter: unquote("progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0077b3', GradientType=0)"); } -.dropdown-menu > .disabled > a, -.dropdown-menu > .disabled > a:hover, -.dropdown-menu > .disabled > a:focus { +.dropdown-menu > .disabled button, +.dropdown-menu > .disabled button:hover, +.dropdown-menu > .disabled button:focus { color: #999999; } -.dropdown-menu > .disabled > a:hover, -.dropdown-menu > .disabled > a:focus { +.dropdown-menu > .disabled button:hover, +.dropdown-menu > .disabled button:focus { text-decoration: none; background-color: transparent; background-image: none; @@ -167,7 +167,7 @@ a.dropdown-toggle{ margin-bottom: -2px; border-radius: 5px 5px 5px 0; } -.dropdown-submenu > a:after { +.dropdown-submenu button:after { display: block; content: " "; float: right; @@ -180,7 +180,7 @@ a.dropdown-toggle{ margin-top: 5px; margin-right: -10px; } -.dropdown-submenu:hover > a:after { +.dropdown-submenu:hover button:after { border-left-color: #ffffff; } .dropdown-submenu.pull-left { diff --git a/scss/modules.scss b/scss/modules.scss index 64f0fbb5f8..3516c0c235 100644 --- a/scss/modules.scss +++ b/scss/modules.scss @@ -103,13 +103,24 @@ a.mini-user:hover { } #members { /* used on accounts elsewhere for GitHub org = Bitbucket team */ - list-style: none; - margin-bottom: 12pt; - margin: 0 auto; + margin: 1em auto 1em; - td { - padding: 0 0 0.5em 0; + td, th { + padding: 0 6px 6px; text-align: left; + vertical-align: baseline; + } + .declines, .not-on-gittip { + font-size: 75%; + a { + font-weight: normal; + } + } + .declines { + color: #aaa; + a { + color: $darker-green; + } } img { width: 18pt; diff --git a/scss/widgets/_sign_in.scss b/scss/widgets/_sign_in.scss index ebc0e023a1..2b9bfa35ca 100644 --- a/scss/widgets/_sign_in.scss +++ b/scss/widgets/_sign_in.scss @@ -1,5 +1,6 @@ .sign-in { display: inline-block; + vertical-align: middle; .dropdown-toggle { background: $green; @include border-radius(3px); @@ -31,28 +32,44 @@ min-width: 120px; li { margin: 0; - &.twitter a { + &.twitter button { @include has-icon("twitter"); } - &.github a { + &.github button { @include has-icon("github"); } - &.bitbucket a { + &.bitbucket button { @include has-icon("bitbucket"); } - &.openstreetmap a { + &.openstreetmap button { @include has-icon("openstreetmap"); } + &:hover button { + background: $darker-green; + } } - a { + button { color: #fff; + display: block; + width: 100%; padding: 3px 8px; + margin: 0; text-align: left; position: relative; font-size: 12px; - &:hover { - background: $darker-green; - } + border: none; + @include border-radius(0); + background: none; + } + button:before { + position: absolute; + top: 4px; + right: 5px; + } + form { + display: block; + margin: 0; + padding: 0; } } #header & { @@ -61,10 +78,5 @@ left: auto; right: 0; } - a:before { - position: absolute; - top: 4px; - right: 5px; - } } } diff --git a/templates/auth.html b/templates/auth.html new file mode 100644 index 0000000000..f82bc98b4c --- /dev/null +++ b/templates/auth.html @@ -0,0 +1,9 @@ +{% macro auth_button(platform, action, user_name='') %} +
+ + + + + +
+{% endmacro %} diff --git a/templates/connected-accounts.html b/templates/connected-accounts.html index 56ec4d9b1b..1898c8936b 100644 --- a/templates/connected-accounts.html +++ b/templates/connected-accounts.html @@ -1,142 +1,35 @@ +{% from 'templates/auth.html' import auth_button with context %} +

Connected Accounts

+{% for platform in website.platforms %} + {% set account = accounts.get(platform.name, None) %} - - - - - - - - - - - - - - - - - - - - +{% endfor %}