Merge tag 'v3.2.1' into merge-3.2.1

CHANGELOG.md

@@ -3,6 +3,35 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
+## [3.2.1] - 2020-10-19
+### Added
+
+- Add support for latest HTTP Signatures spec draft ([ThibG](https://github.com/tootsuite/mastodon/pull/14556))
+- Add support for inlined objects in ActivityPub `to`/`cc` ([ThibG](https://github.com/tootsuite/mastodon/pull/14514))
+
+### Changed
+
+- Change actors to not be served at all without authentication in limited federation mode ([ThibG](https://github.com/tootsuite/mastodon/pull/14800))
+  - Previously, a bare version of an actor was served when not authenticated, i.e. username and public key
+  - Because all actor fetch requests are signed using a separate system actor, that is no longer required
+
+### Fixed
+
+- Fix `tootctl media` commands not recognizing very large IDs ([ThibG](https://github.com/tootsuite/mastodon/pull/14536))
+- Fix crash when failing to load emoji picker in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/14525))
+- Fix contrast requirements in thumbnail color extraction ([ThibG](https://github.com/tootsuite/mastodon/pull/14464))
+- Fix audio/video player not using `CDN_HOST` on public pages ([ThibG](https://github.com/tootsuite/mastodon/pull/14486))
+- Fix private boost icon not being used on public pages ([OmmyZhang](https://github.com/tootsuite/mastodon/pull/14471))
+- Fix audio player on Safari in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/14485), [ThibG](https://github.com/tootsuite/mastodon/pull/14465))
+- Fix dereferencing remote statuses not using the correct account for signature when receiving a targeted inbox delivery ([ThibG](https://github.com/tootsuite/mastodon/pull/14656))
+- Fix nil error in `tootctl media remove` ([noellabo](https://github.com/tootsuite/mastodon/pull/14657))
+- Fix videos with near-60 fps being rejected ([Gargron](https://github.com/tootsuite/mastodon/pull/14684))
+- Fix reported statuses not being included in warning e-mail ([Gargron](https://github.com/tootsuite/mastodon/pull/14778))
+- Fix `Reject` activities of `Follow` objects not correctly destroying a follow relationship ([ThibG](https://github.com/tootsuite/mastodon/pull/14479))
+- Fix inefficiencies in fan-out-on-write service ([Gargron](https://github.com/tootsuite/mastodon/pull/14682), [noellabo](https://github.com/tootsuite/mastodon/pull/14709))
+- Fix timeout errors when trying to webfinger some IPv6 configurations ([Gargron](https://github.com/tootsuite/mastodon/pull/14919))
+- Fix files served as `application/octet-stream` being rejected without attempting mime type detection ([ThibG](https://github.com/tootsuite/mastodon/pull/14452))
+
 ## [3.2.0] - 2020-07-27
 ### Added
 

Gemfile

@@ -54,7 +54,6 @@ gem 'doorkeeper', '~> 5.4'
 gem 'ed25519', '~> 1.2'
 gem 'fast_blank', '~> 1.0'
 gem 'fastimage'
-gem 'goldfinger', '~> 2.1'
 gem 'hiredis', '~> 0.6'
 gem 'redis-namespace', '~> 1.7'
 gem 'health_check', git: 'https://github.com/ianheggie/health_check', ref: '0b799ead604f900ed50685e9b2d469cd2befba5b'

Gemfile.lock

@@ -252,11 +252,6 @@ GEM
       ruby-progressbar (~> 1.4)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    goldfinger (2.1.1)
-      addressable (~> 2.5)
-      http (~> 4.0)
-      nokogiri (~> 1.8)
-      oj (~> 3.0)
     hamlit (2.11.0)
       temple (>= 0.8.2)
       thor
@@ -711,7 +706,6 @@ DEPENDENCIES
   fog-core (<= 2.1.0)
   fog-openstack (~> 0.3)
   fuubar (~> 2.5)
-  goldfinger (~> 2.1)
   hamlit-rails (~> 0.2)
   health_check!
   hiredis (~> 0.6)

app/controllers/accounts_controller.rb

@@ -7,6 +7,7 @@ class AccountsController < ApplicationController
   include AccountControllerConcern
   include SignatureAuthentication
 
+  before_action :require_signature!, if: -> { request.format == :json && authorized_fetch_mode? }
   before_action :set_cache_headers
   before_action :set_body_classes
 
@@ -49,7 +50,7 @@ def show
 
       format.json do
         expires_in 3.minutes, public: !(authorized_fetch_mode? && signed_request_account.present?)
-        render_with_cache json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter, fields: restrict_fields_to
+        render_with_cache json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter
       end
     end
   end
@@ -149,12 +150,4 @@ def filtered_status_page
   def params_slice(*keys)
     params.slice(*keys).permit(*keys)
   end
-
-  def restrict_fields_to
-    if signed_request_account.present? || public_fetch_mode?
-      # Return all fields
-    else
-      %i(id type preferred_username inbox public_key endpoints)
-    end
-  end
 end

app/controllers/concerns/signature_verification.rb

@@ -7,6 +7,44 @@ module SignatureVerification
 
   include DomainControlHelper
 
+  EXPIRATION_WINDOW_LIMIT = 12.hours
+  CLOCK_SKEW_MARGIN       = 1.hour
+
+  class SignatureVerificationError < StandardError; end
+
+  class SignatureParamsParser < Parslet::Parser
+    rule(:token)         { match("[0-9a-zA-Z!#$%&'*+.^_`|~-]").repeat(1).as(:token) }
+    rule(:quoted_string) { str('"') >> (qdtext | quoted_pair).repeat.as(:quoted_string) >> str('"') }
+    # qdtext and quoted_pair are not exactly according to spec but meh
+    rule(:qdtext)        { match('[^\\\\"]') }
+    rule(:quoted_pair)   { str('\\') >> any }
+    rule(:bws)           { match('\s').repeat }
+    rule(:param)         { (token.as(:key) >> bws >> str('=') >> bws >> (token | quoted_string).as(:value)).as(:param) }
+    rule(:comma)         { bws >> str(',') >> bws }
+    # Old versions of node-http-signature add an incorrect "Signature " prefix to the header
+    rule(:buggy_prefix)  { str('Signature ') }
+    rule(:params)        { buggy_prefix.maybe >> (param >> (comma >> param).repeat).as(:params) }
+    root(:params)
+  end
+
+  class SignatureParamsTransformer < Parslet::Transform
+    rule(params: subtree(:p)) do
+      (p.is_a?(Array) ? p : [p]).each_with_object({}) { |(key, val), h| h[key] = val }
+    end
+
+    rule(param: { key: simple(:key), value: simple(:val) }) do
+      [key, val]
+    end
+
+    rule(quoted_string: simple(:string)) do
+      string.to_s
+    end
+
+    rule(token: simple(:string)) do
+      string.to_s
+    end
+  end
+
   def require_signature!
     render plain: signature_verification_failure_reason, status: signature_verification_failure_code unless signed_request_account
   end
@@ -24,72 +62,40 @@ def signature_verification_failure_code
   end
 
   def signature_key_id
-    raw_signature    = request.headers['Signature']
-    signature_params = {}
-
-    raw_signature.split(',').each do |part|
-      parsed_parts = part.match(/([a-z]+)="([^"]+)"/i)
-      next if parsed_parts.nil? || parsed_parts.size != 3
-      signature_params[parsed_parts[1]] = parsed_parts[2]
-    end
-
     signature_params['keyId']
+  rescue SignatureVerificationError
+    nil
   end
 
   def signed_request_account
     return @signed_request_account if defined?(@signed_request_account)
 
-    unless signed_request?
-      @signature_verification_failure_reason = 'Request not signed'
-      @signed_request_account = nil
-      return
-    end
-
-    if request.headers['Date'].present? && !matches_time_window?
-      @signature_verification_failure_reason = 'Signed request date outside acceptable time window'
-      @signed_request_account = nil
-      return
-    end
+    raise SignatureVerificationError, 'Request not signed' unless signed_request?
+    raise SignatureVerificationError, 'Incompatible request signature. keyId and signature are required' if missing_required_signature_parameters?
+    raise SignatureVerificationError, 'Unsupported signature algorithm (only rsa-sha256 and hs2019 are supported)' unless %w(rsa-sha256 hs2019).include?(signature_algorithm)
+    raise SignatureVerificationError, 'Signed request date outside acceptable time window' unless matches_time_window?
 
-    raw_signature    = request.headers['Signature']
-    signature_params = {}
-
-    raw_signature.split(',').each do |part|
-      parsed_parts = part.match(/([a-z]+)="([^"]+)"/i)
-      next if parsed_parts.nil? || parsed_parts.size != 3
-      signature_params[parsed_parts[1]] = parsed_parts[2]
-    end
-
-    if incompatible_signature?(signature_params)
-      @signature_verification_failure_reason = 'Incompatible request signature'
-      @signed_request_account = nil
-      return
-    end
+    verify_signature_strength!
 
     account = account_from_key_id(signature_params['keyId'])
 
-    if account.nil?
-      @signature_verification_failure_reason = "Public key not found for key #{signature_params['keyId']}"
-      @signed_request_account = nil
-      return
-    end
+    raise SignatureVerificationError, "Public key not found for key #{signature_params['keyId']}" if account.nil?
 
     signature             = Base64.decode64(signature_params['signature'])
-    compare_signed_string = build_signed_string(signature_params['headers'])
+    compare_signed_string = build_signed_string
 
     return account unless verify_signature(account, signature, compare_signed_string).nil?
 
     account = stoplight_wrap_request { account.possibly_stale? ? account.refresh! : account_refresh_key(account) }
 
-    if account.nil?
-      @signature_verification_failure_reason = "Public key not found for key #{signature_params['keyId']}"
-      @signed_request_account = nil
-      return
-    end
+    raise SignatureVerificationError, "Public key not found for key #{signature_params['keyId']}" if account.nil?
 
     return account unless verify_signature(account, signature, compare_signed_string).nil?
 
-    @signature_verification_failure_reason = "Verification failed for #{account.username}@#{account.domain} #{account.uri}"
+    @signature_verification_failure_reason = "Verification failed for #{account.username}@#{account.domain} #{account.uri} using rsa-sha256 (RSASSA-PKCS1-v1_5 with SHA-256)"
+    @signed_request_account = nil
+  rescue SignatureVerificationError => e
+    @signature_verification_failure_reason = e.message
     @signed_request_account = nil
   end
 
@@ -99,6 +105,31 @@ def request_body
 
   private
 
+  def signature_params
+    @signature_params ||= begin
+      raw_signature = request.headers['Signature']
+      tree          = SignatureParamsParser.new.parse(raw_signature)
+      SignatureParamsTransformer.new.apply(tree)
+    end
+  rescue Parslet::ParseFailed
+    raise SignatureVerificationError, 'Error parsing signature parameters'
+  end
+
+  def signature_algorithm
+    signature_params.fetch('algorithm', 'hs2019')
+  end
+
+  def signed_headers
+    signature_params.fetch('headers', signature_algorithm == 'hs2019' ? '(created)' : 'date').downcase.split(' ')
+  end
+
+  def verify_signature_strength!
+    raise SignatureVerificationError, 'Mastodon requires the Date header or (created) pseudo-header to be signed' unless signed_headers.include?('date') || signed_headers.include?('(created)')
+    raise SignatureVerificationError, 'Mastodon requires the Digest header or (request-target) pseudo-header to be signed' unless signed_headers.include?(Request::REQUEST_TARGET) || signed_headers.include?('digest')
+    raise SignatureVerificationError, 'Mastodon requires the Host header to be signed' unless signed_headers.include?('host')
+    raise SignatureVerificationError, 'Mastodon requires the Digest header to be signed when doing a POST request' if request.post? && !signed_headers.include?('digest')
+  end
+
   def verify_signature(account, signature, compare_signed_string)
     if account.keypair.public_key.verify(OpenSSL::Digest::SHA256.new, signature, compare_signed_string)
       @signed_request_account = account
@@ -108,12 +139,20 @@ def verify_signature(account, signature, compare_signed_string)
     nil
   end
 
-  def build_signed_string(signed_headers)
-    signed_headers = 'date' if signed_headers.blank?
-
-    signed_headers.downcase.split(' ').map do |signed_header|
+  def build_signed_string
+    signed_headers.map do |signed_header|
       if signed_header == Request::REQUEST_TARGET
         "#{Request::REQUEST_TARGET}: #{request.method.downcase} #{request.path}"
+      elsif signed_header == '(created)'
+        raise SignatureVerificationError, 'Invalid pseudo-header (created) for rsa-sha256' unless signature_algorithm == 'hs2019'
+        raise SignatureVerificationError, 'Pseudo-header (created) used but corresponding argument missing' if signature_params['created'].blank?
+
+        "(created): #{signature_params['created']}"
+      elsif signed_header == '(expires)'
+        raise SignatureVerificationError, 'Invalid pseudo-header (expires) for rsa-sha256' unless signature_algorithm == 'hs2019'
+        raise SignatureVerificationError, 'Pseudo-header (expires) used but corresponding argument missing' if signature_params['expires'].blank?
+
+        "(expires): #{signature_params['expires']}"
       elsif signed_header == 'digest'
         "digest: #{body_digest}"
       else
@@ -123,13 +162,28 @@ def build_signed_string(signed_headers)
   end
 
   def matches_time_window?
+    created_time = nil
+    expires_time = nil
+
     begin
-      time_sent = Time.httpdate(request.headers['Date'])
+      if signature_algorithm == 'hs2019' && signature_params['created'].present?
+        created_time = Time.at(signature_params['created'].to_i).utc
+      elsif request.headers['Date'].present?
+        created_time = Time.httpdate(request.headers['Date']).utc
+      end
+
+      expires_time = Time.at(signature_params['expires'].to_i).utc if signature_params['expires'].present?
     rescue ArgumentError
       return false
     end
 
-    (Time.now.utc - time_sent).abs <= 12.hours
+    expires_time ||= created_time + 5.minutes unless created_time.nil?
+    expires_time = [expires_time, created_time + EXPIRATION_WINDOW_LIMIT].min unless created_time.nil?
+
+    return false if created_time.present? && created_time > Time.now.utc + CLOCK_SKEW_MARGIN
+    return false if expires_time.present? && Time.now.utc > expires_time + CLOCK_SKEW_MARGIN
+
+    true
   end
 
   def body_digest
@@ -140,9 +194,8 @@ def to_header_name(name)
     name.split(/-/).map(&:capitalize).join('-')
   end
 
-  def incompatible_signature?(signature_params)
-    signature_params['keyId'].blank? ||
-      signature_params['signature'].blank?
+  def missing_required_signature_parameters?
+    signature_params['keyId'].blank? || signature_params['signature'].blank?
   end
 
   def account_from_key_id(key_id)

app/helpers/webfinger_helper.rb

@@ -1,38 +1,7 @@
 # frozen_string_literal: true
 
-# Monkey-patch on monkey-patch.
-# Because it conflicts with the request.rb patch.
-class HTTP::Timeout::PerOperationOriginal < HTTP::Timeout::PerOperation
-  def connect(socket_class, host, port, nodelay = false)
-    ::Timeout.timeout(@connect_timeout, HTTP::TimeoutError) do
-      @socket = socket_class.open(host, port)
-      @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay
-    end
-  end
-end
-
 module WebfingerHelper
   def webfinger!(uri)
-    hidden_service_uri = /\.(onion|i2p)(:\d+)?$/.match(uri)
-
-    raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if !Rails.configuration.x.access_to_hidden_service && hidden_service_uri
-
-    opts = {
-      ssl: !hidden_service_uri,
-
-      headers: {
-        'User-Agent': Mastodon::Version.user_agent,
-      },
-
-      timeout_class: HTTP::Timeout::PerOperationOriginal,
-
-      timeout_options: {
-        write_timeout: 10,
-        connect_timeout: 5,
-        read_timeout: 10,
-      },
-    }
-
-    Goldfinger::Client.new(uri, opts.merge(Rails.configuration.x.http_client_proxy)).finger
+    Webfinger.new(uri).perform
   end
 end

app/javascript/mastodon/components/status_action_bar.js

@@ -7,6 +7,7 @@ import DropdownMenuContainer from '../containers/dropdown_menu_container';
 import { defineMessages, injectIntl } from 'react-intl';
 import ImmutablePureComponent from 'react-immutable-pure-component';
 import { me, isStaff } from '../initial_state';
+import classNames from 'classnames';
 
 const messages = defineMessages({
   delete: { id: 'status.delete', defaultMessage: 'Delete' },
@@ -329,7 +330,7 @@ class StatusActionBar extends ImmutablePureComponent {
     return (
       <div className='status__action-bar'>
         <div className='status__action-bar__counter'><IconButton className='status__action-bar-button' title={replyTitle} icon={status.get('in_reply_to_account_id') === status.getIn(['account', 'id']) ? 'reply' : replyIcon} onClick={this.handleReplyClick} /><span className='status__action-bar__counter__label' >{obfuscatedCount(status.get('replies_count'))}</span></div>
-        <IconButton className='status__action-bar-button' disabled={!publicStatus && !reblogPrivate} active={status.get('reblogged')} pressed={status.get('reblogged')} title={reblogTitle} icon='retweet' onClick={this.handleReblogClick} />
+        <IconButton className={classNames('status__action-bar-button', { reblogPrivate })} disabled={!publicStatus && !reblogPrivate}  active={status.get('reblogged')} pressed={status.get('reblogged')} title={reblogTitle} icon='retweet' onClick={this.handleReblogClick} />
         <IconButton className='status__action-bar-button star-icon' animate active={status.get('favourited')} pressed={status.get('favourited')} title={intl.formatMessage(messages.favourite)} icon='star' onClick={this.handleFavouriteClick} />
         {shareButton}