Bump version to 3.4.10

* Fix front-end emoji tests

* Fix nodes order being sometimes mangled when rewriting emoji

.circleci/config.yml

@@ -167,8 +167,45 @@ jobs:
           name: Create database
           command: ./bin/rails db:create
       - run:
-          name: Run migrations
+          command: ./bin/rails db:migrate VERSION=20171010025614
+          name: Run migrations up to v2.0.0
+      - run:
+          command: ./bin/rails tests:migrations:populate_v2
+          name: Populate database with test data
+      - run:
           command: ./bin/rails db:migrate
+          name: Run all remaining migrations
+
+  test-two-step-migrations:
+    <<: *defaults
+    docker:
+      - image: circleci/ruby:2.7-buster-node
+        environment: *ruby_environment
+      - image: circleci/postgres:12.2
+        environment:
+          POSTGRES_USER: root
+          POSTGRES_HOST_AUTH_METHOD: trust
+      - image: circleci/redis:5-alpine
+    steps:
+      - *attach_workspace
+      - *install_system_dependencies
+      - run:
+          command: ./bin/rails db:create
+          name: Create database
+      - run:
+          command: ./bin/rails db:migrate VERSION=20171010025614
+          name: Run migrations up to v2.0.0
+      - run:
+          command: ./bin/rails tests:migrations:populate_v2
+          name: Populate database with test data
+      - run:
+          command: ./bin/rails db:migrate
+          name: Run all pre-deployment migrations
+          evironment:
+            SKIP_POST_DEPLOYMENT_MIGRATIONS: true
+      - run:
+          command: ./bin/rails db:migrate
+          name: Run all post-deployment remaining migrations
 
   test-ruby2.7:
     <<: *defaults
@@ -238,6 +275,9 @@ workflows:
       - test-migrations:
           requires:
             - install-ruby2.7
+      - test-two-step-migrations:
+          requires:
+            - install-ruby2.7
       - test-ruby2.7:
           requires:
             - install-ruby2.7

.env.production.sample

@@ -4,6 +4,12 @@
 # not demonstrate all available configuration options. Please look at
 # https://docs.joinmastodon.org/admin/config/ for the full documentation.
 
+# Note that this file accepts slightly different syntax depending on whether
+# you are using `docker-compose` or not. In particular, if you use
+# `docker-compose`, the value of each declared variable will be taken verbatim,
+# including surrounding quotes.
+# See: https://github.com/mastodon/mastodon/issues/16895
+
 # Federation
 # ----------
 # This identifies your server and cannot be changed safely later

.github/CODEOWNERS

@@ -1,4 +1,4 @@
-# CODEOWNERS for tootsuite/mastodon
+# CODEOWNERS for mastodon/mastodon
 
 # Translators
 # To add translator, copy these lines, replace `fr` with appropriate language code and replace `@żelipapą` with user's GitHub nickname preceded by `@` sign or e-mail address.

.github/workflows/build-image.yml

@@ -0,0 +1,34 @@
+name: Build container image
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "main"
+    tags:
+      - "*"
+jobs:
+  build-image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - uses: docker/metadata-action@v3
+        id: meta
+        with:
+          images: tootsuite/mastodon
+          flavor: |
+            latest=auto
+          tags: |
+            type=edge,branch=main
+            type=semver,pattern={{ raw }}
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=registry,ref=tootsuite/mastodon:latest
+          cache-to: type=inline

AUTHORS.md

@@ -1,7 +1,7 @@
 Authors
 =======
 
-Mastodon is available on [GitHub](https://github.com/tootsuite/mastodon)
+Mastodon is available on [GitHub](https://github.com/mastodon/mastodon)
 and provided thanks to the work of the following contributors:
 
 * [Gargron](https://github.com/Gargron)
@@ -719,7 +719,7 @@ and provided thanks to the work of the following contributors:
 * [西小倉宏信](mailto:nishiko@mindia.jp)
 * [雨宮美羽](mailto:k737566@gmail.com)
 
-This document is provided for informational purposes only. Since it is only updated once per release, the version you are looking at may be currently out of date. To see the full list of contributors, consider looking at the [git history](https://github.com/tootsuite/mastodon/graphs/contributors) instead.
+This document is provided for informational purposes only. Since it is only updated once per release, the version you are looking at may be currently out of date. To see the full list of contributors, consider looking at the [git history](https://github.com/mastodon/mastodon/graphs/contributors) instead.
 
 ## Translators
 

CONTRIBUTING.md

@@ -14,7 +14,7 @@ If your contributions are accepted into Mastodon, you can request to be paid thr
 
 ## Bug reports
 
-Bug reports and feature suggestions must use descriptive and concise titles and be submitted to [GitHub Issues](https://github.com/tootsuite/mastodon/issues). Please use the search function to make sure that you are not submitting duplicates, and that a similar report or request has not already been resolved or rejected.
+Bug reports and feature suggestions must use descriptive and concise titles and be submitted to [GitHub Issues](https://github.com/mastodon/mastodon/issues). Please use the search function to make sure that you are not submitting duplicates, and that a similar report or request has not already been resolved or rejected.
 
 ## Translations
 
@@ -44,4 +44,4 @@ It is not always possible to phrase every change in such a manner, but it is des
 
 ## Documentation
 
-The [Mastodon documentation](https://docs.joinmastodon.org) is a statically generated site. You can [submit merge requests to tootsuite/documentation](https://github.com/tootsuite/documentation).
+The [Mastodon documentation](https://docs.joinmastodon.org) is a statically generated site. You can [submit merge requests to mastodon/documentation](https://github.com/mastodon/documentation).

Dockerfile

@@ -54,8 +54,8 @@ RUN npm install -g yarn && \
 COPY Gemfile* package.json yarn.lock /opt/mastodon/
 
 RUN cd /opt/mastodon && \
-  bundle config set deployment 'true' && \
-  bundle config set without 'development test' && \
+  bundle config set --local deployment 'true' && \
+  bundle config set --local without 'development test' && \
 	bundle install -j"$(nproc)" && \
 	yarn install --pure-lockfile
 

Gemfile.lock

@@ -545,8 +545,9 @@ GEM
       rack (>= 1.1)
       rubocop (>= 1.7.0, < 2.0)
     ruby-progressbar (1.11.0)
-    ruby-saml (1.11.0)
-      nokogiri (>= 1.5.10)
+    ruby-saml (1.13.0)
+      nokogiri (>= 1.10.5)
+      rexml
     ruby2_keywords (0.0.4)
     rufus-scheduler (3.6.0)
       fugit (~> 1.1, >= 1.1.6)

README.md

@@ -1,14 +1,14 @@
 ![Mastodon](https://i.imgur.com/NhZc40l.png)
 ========
 
-[![GitHub release](https://img.shields.io/github/release/tootsuite/mastodon.svg)][releases]
-[![Build Status](https://img.shields.io/circleci/project/github/tootsuite/mastodon.svg)][circleci]
+[![GitHub release](https://img.shields.io/github/release/mastodon/mastodon.svg)][releases]
+[![Build Status](https://img.shields.io/circleci/project/github/mastodon/mastodon.svg)][circleci]
 [![Code Climate](https://img.shields.io/codeclimate/maintainability/tootsuite/mastodon.svg)][code_climate]
 [![Crowdin](https://d322cqt584bo4o.cloudfront.net/mastodon/localized.svg)][crowdin]
 [![Docker Pulls](https://img.shields.io/docker/pulls/tootsuite/mastodon.svg)][docker]
 
-[releases]: https://github.com/tootsuite/mastodon/releases
-[circleci]: https://circleci.com/gh/tootsuite/mastodon
+[releases]: https://github.com/mastodon/mastodon/releases
+[circleci]: https://circleci.com/gh/mastodon/mastodon
 [code_climate]: https://codeclimate.com/github/tootsuite/mastodon
 [crowdin]: https://crowdin.com/project/mastodon
 [docker]: https://hub.docker.com/r/tootsuite/mastodon/

app.json

@@ -1,8 +1,8 @@
 {
   "name": "Mastodon",
   "description": "A GNU Social-compatible microblogging server",
-  "repository": "https://github.com/tootsuite/mastodon",
-  "logo": "https://github.com/tootsuite.png",
+  "repository": "https://github.com/mastodon/mastodon",
+  "logo": "https://github.com/mastodon.png",
   "env": {
     "HEROKU": {
       "description": "Leave this as true",

app/controllers/activitypub/base_controller.rb

@@ -2,6 +2,7 @@
 
 class ActivityPub::BaseController < Api::BaseController
   skip_before_action :require_authenticated_user!
+  skip_before_action :require_not_suspended!
 
   private
 

app/controllers/activitypub/followers_synchronizations_controller.rb

@@ -19,11 +19,11 @@ class ActivityPub::FollowersSynchronizationsController < ActivityPub::BaseContro
   private
 
   def uri_prefix
-    signed_request_account.uri[/http(s?):\/\/[^\/]+\//]
+    signed_request_account.uri[Account::URL_PREFIX_RE]
   end
 
   def set_items
-    @items = @account.followers.where(Account.arel_table[:uri].matches(uri_prefix + '%', false, true)).pluck(:uri)
+    @items = @account.followers.where(Account.arel_table[:uri].matches("#{Account.sanitize_sql_like(uri_prefix)}/%", false, true)).or(@account.followers.where(uri: uri_prefix)).pluck(:uri)
   end
 
   def collection_presenter

app/controllers/activitypub/outboxes_controller.rb

@@ -11,7 +11,11 @@ class ActivityPub::OutboxesController < ActivityPub::BaseController
   before_action :set_cache_headers
 
   def show
-    expires_in(page_requested? ? 0 : 3.minutes, public: public_fetch_mode? && !(signed_request_account.present? && page_requested?))
+    if page_requested?
+      expires_in(1.minute, public: public_fetch_mode? && signed_request_account.nil?)
+    else
+      expires_in(3.minutes, public: public_fetch_mode?)
+    end
     render json: outbox_presenter, serializer: ActivityPub::OutboxSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json'
   end
 
@@ -76,4 +80,8 @@ class ActivityPub::OutboxesController < ActivityPub::BaseController
   def set_account
     @account = params[:account_username].present? ? Account.find_local!(username_param) : Account.representative
   end
+
+  def set_cache_headers
+    response.headers['Vary'] = 'Signature' if authorized_fetch_mode? || page_requested?
+  end
 end

app/controllers/admin/statuses_controller.rb

@@ -14,7 +14,7 @@ module Admin
       @statuses = @account.statuses.where(visibility: [:public, :unlisted])
 
       if params[:media]
-        @statuses.merge!(Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id))
+        @statuses.merge!(Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id)).reorder('statuses.id desc')
       end
 
       @statuses = @statuses.preload(:media_attachments, :mentions).page(params[:page]).per(PER_PAGE)

app/controllers/api/base_controller.rb

@@ -10,6 +10,7 @@ class Api::BaseController < ApplicationController
   skip_before_action :require_functional!, unless: :whitelist_mode?
 
   before_action :require_authenticated_user!, if: :disallow_unauthenticated_api_access?
+  before_action :require_not_suspended!
   before_action :set_cache_headers
 
   protect_from_forgery with: :null_session
@@ -93,6 +94,10 @@ class Api::BaseController < ApplicationController
     render json: { error: 'This method requires an authenticated user' }, status: 401 unless current_user
   end
 
+  def require_not_suspended!
+    render json: { error: 'Your login is currently disabled' }, status: 403 if current_user&.account&.suspended?
+  end
+
   def require_user!
     if !current_user
       render json: { error: 'This method requires an authenticated user' }, status: 422

app/controllers/api/v1/reports_controller.rb

@@ -21,7 +21,7 @@ class Api::V1::ReportsController < Api::BaseController
   private
 
   def reported_status_ids
-    reported_account.statuses.with_discarded.find(status_ids).pluck(:id)
+    reported_account.statuses.with_discarded.permitted_for(reported_account, current_account).find(status_ids).pluck(:id)
   end
 
   def status_ids

app/controllers/auth/confirmations_controller.rb

@@ -40,7 +40,7 @@ class Auth::ConfirmationsController < Devise::ConfirmationsController
 
   def after_confirmation_path_for(_resource_name, user)
     if user.created_by_application && truthy_param?(:redirect_to_app)
-      user.created_by_application.redirect_uri
+      user.created_by_application.confirmation_redirect_uri
     else
       super
     end

app/controllers/auth/passwords_controller.rb

@@ -10,7 +10,6 @@ class Auth::PasswordsController < Devise::PasswordsController
     super do |resource|
       if resource.errors.empty?
         resource.session_activations.destroy_all
-        resource.forget_me!
       end
     end
   end

app/controllers/auth/registrations_controller.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 class Auth::RegistrationsController < Devise::RegistrationsController
-  include Devise::Controllers::Rememberable
   include RegistrationSpamConcern
 
   layout :determine_layout
@@ -30,8 +29,6 @@ class Auth::RegistrationsController < Devise::RegistrationsController
     super do |resource|
       if resource.saved_change_to_encrypted_password?
         resource.clear_other_sessions(current_session.session_id)
-        resource.forget_me!
-        remember_me(resource)
       end
     end
   end

app/controllers/auth/sessions_controller.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 class Auth::SessionsController < Devise::SessionsController
-  include Devise::Controllers::Rememberable
-
   layout 'auth'
 
   skip_before_action :require_no_authentication, only: [:create]
@@ -26,7 +24,6 @@ class Auth::SessionsController < Devise::SessionsController
   def create
     super do |resource|
       resource.update_sign_in!(request, new_sign_in: true)
-      remember_me(resource)
       flash.delete(:notice)
     end
   end
@@ -40,7 +37,7 @@ class Auth::SessionsController < Devise::SessionsController
   end
 
   def webauthn_options
-    user = find_user
+    user = User.find_by(id: session[:attempt_user_id])
 
     if user.webauthn_enabled?
       options_for_get = WebAuthn::Credential.options_for_get(
@@ -58,16 +55,20 @@ class Auth::SessionsController < Devise::SessionsController
   protected
 
   def find_user
-    if session[:attempt_user_id]
+    if user_params[:email].present?
+      find_user_from_params
+    elsif session[:attempt_user_id]
       User.find_by(id: session[:attempt_user_id])
-    else
-      user   = User.authenticate_with_ldap(user_params) if Devise.ldap_authentication
-      user ||= User.authenticate_with_pam(user_params) if Devise.pam_authentication
-      user ||= User.find_for_authentication(email: user_params[:email])
-      user
     end
   end
 
+  def find_user_from_params
+    user   = User.authenticate_with_ldap(user_params) if Devise.ldap_authentication
+    user ||= User.authenticate_with_pam(user_params) if Devise.pam_authentication
+    user ||= User.find_for_authentication(email: user_params[:email])
+    user
+  end
+
   def user_params
     params.require(:user).permit(:email, :password, :otp_attempt, :sign_in_token_attempt, credential: {})
   end

app/controllers/concerns/sign_in_token_authentication_concern.rb

@@ -16,21 +16,24 @@ module SignInTokenAuthenticationConcern
   end
 
   def authenticate_with_sign_in_token
-    user = self.resource = find_user
+    if user_params[:email].present?
+      user = self.resource = find_user_from_params
+      prompt_for_sign_in_token(user) if user&.external_or_valid_password?(user_params[:password])
+    elsif session[:attempt_user_id]
+      user = self.resource = User.find_by(id: session[:attempt_user_id])
+      return if user.nil?
 
-    if user.present? && session[:attempt_user_id].present? && session[:attempt_user_updated_at] != user.updated_at.to_s
-      restart_session
-    elsif user_params.key?(:sign_in_token_attempt) && session[:attempt_user_id]
-      authenticate_with_sign_in_token_attempt(user)
-    elsif user.present? && user.external_or_valid_password?(user_params[:password])
-      prompt_for_sign_in_token(user)
+      if session[:attempt_user_updated_at] != user.updated_at.to_s
+        restart_session
+      elsif user_params.key?(:sign_in_token_attempt)
+        authenticate_with_sign_in_token_attempt(user)
+      end
     end
   end
 
   def authenticate_with_sign_in_token_attempt(user)
     if valid_sign_in_token_attempt?(user)
       clear_attempt_from_session
-      remember_me(user)
       sign_in(user)
     else
       flash.now[:alert] = I18n.t('users.invalid_sign_in_token')

app/controllers/concerns/two_factor_authentication_concern.rb

@@ -35,16 +35,20 @@ module TwoFactorAuthenticationConcern
   end
 
   def authenticate_with_two_factor
-    user = self.resource = find_user
+    if user_params[:email].present?
+      user = self.resource = find_user_from_params
+      prompt_for_two_factor(user) if user&.external_or_valid_password?(user_params[:password])
+    elsif session[:attempt_user_id]
+      user = self.resource = User.find_by(id: session[:attempt_user_id])
+      return if user.nil?
 
-    if user.present? && session[:attempt_user_id].present? && session[:attempt_user_updated_at] != user.updated_at.to_s
-      restart_session
-    elsif user.webauthn_enabled? && user_params.key?(:credential) && session[:attempt_user_id]
-      authenticate_with_two_factor_via_webauthn(user)
-    elsif user_params.key?(:otp_attempt) && session[:attempt_user_id]
-      authenticate_with_two_factor_via_otp(user)
-    elsif user.present? && user.external_or_valid_password?(user_params[:password])
-      prompt_for_two_factor(user)
+      if session[:attempt_user_updated_at] != user.updated_at.to_s
+        restart_session
+      elsif user.webauthn_enabled? && user_params.key?(:credential)
+        authenticate_with_two_factor_via_webauthn(user)
+      elsif user_params.key?(:otp_attempt)
+        authenticate_with_two_factor_via_otp(user)
+      end
     end
   end
 
@@ -53,7 +57,6 @@ module TwoFactorAuthenticationConcern
 
     if valid_webauthn_credential?(user, webauthn_credential)
       clear_attempt_from_session
-      remember_me(user)
       sign_in(user)
       render json: { redirect_path: root_path }, status: :ok
     else
@@ -64,7 +67,6 @@ module TwoFactorAuthenticationConcern
   def authenticate_with_two_factor_via_otp(user)
     if valid_otp_attempt?(user)
       clear_attempt_from_session
-      remember_me(user)
       sign_in(user)
     else
       flash.now[:alert] = I18n.t('users.invalid_otp_token')

app/controllers/follower_accounts_controller.rb

@@ -85,7 +85,7 @@ class FollowerAccountsController < ApplicationController
     if page_requested? || !@account.user_hides_network?
       # Return all fields
     else
-      %i(id type totalItems)
+      %i(id type total_items)
     end
   end
 end

app/controllers/following_accounts_controller.rb

@@ -85,7 +85,7 @@ class FollowingAccountsController < ApplicationController
     if page_requested? || !@account.user_hides_network?
       # Return all fields
     else
-      %i(id type totalItems)
+      %i(id type total_items)
     end
   end
 end

app/controllers/settings/deletes_controller.rb

@@ -42,7 +42,7 @@ class Settings::DeletesController < Settings::BaseController
   end
 
   def destroy_account!
-    current_account.suspend!(origin: :local)
+    current_account.suspend!(origin: :local, block_email: false)
     AccountDeletionWorker.perform_async(current_user.account_id)
     sign_out
   end

app/controllers/well_known/webfinger_controller.rb

@@ -4,7 +4,6 @@ module WellKnown
   class WebfingerController < ActionController::Base
     include RoutingHelper
 
-    before_action { response.headers['Vary'] = 'Accept' }
     before_action :set_account
     before_action :check_account_suspension
 
@@ -39,10 +38,12 @@ module WellKnown
     end
 
     def bad_request
+      expires_in(3.minutes, public: true)
       head 400
     end
 
     def not_found
+      expires_in(3.minutes, public: true)
       head 404
     end
 

app/helpers/accounts_helper.rb

@@ -80,17 +80,17 @@ module AccountsHelper
   def account_description(account)
     prepend_str = [
       [
-        number_to_human(account.statuses_count, strip_insignificant_zeros: true),
+        number_to_human(account.statuses_count, precision: 3, strip_insignificant_zeros: true),
         I18n.t('accounts.posts', count: account.statuses_count),
       ].join(' '),
 
       [
-        number_to_human(account.following_count, strip_insignificant_zeros: true),
+        number_to_human(account.following_count, precision: 3, strip_insignificant_zeros: true),
         I18n.t('accounts.following', count: account.following_count),
       ].join(' '),
 
       [
-        number_to_human(account.followers_count, strip_insignificant_zeros: true),
+        number_to_human(account.followers_count, precision: 3, strip_insignificant_zeros: true),
         I18n.t('accounts.followers', count: account.followers_count),
       ].join(' '),
     ].join(', ')

app/helpers/application_helper.rb

@@ -14,6 +14,17 @@ module ApplicationHelper
     ku
   ).freeze
 
+  def friendly_number_to_human(number, **options)
+    # By default, the number of precision digits used by number_to_human
+    # is looked up from the locales definition, and rails-i18n comes with
+    # values that don't seem to make much sense for many languages, so
+    # override these values with a default of 3 digits of precision.
+    options[:precision] = 3
+    options[:strip_insignificant_zeros] = true
+
+    number_to_human(number, **options)
+  end
+
   def active_nav_class(*paths)
     paths.any? { |path| current_page?(path) } ? 'active' : ''
   end

app/helpers/context_helper.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module ContextHelper
+  NAMED_CONTEXT_MAP = {
+    activitystreams: 'https://www.w3.org/ns/activitystreams',
+    security: 'https://w3id.org/security/v1',
+  }.freeze
+
+  CONTEXT_EXTENSION_MAP = {
+    manually_approves_followers: { 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers' },
+    sensitive: { 'sensitive' => 'as:sensitive' },
+    hashtag: { 'Hashtag' => 'as:Hashtag' },
+    moved_to: { 'movedTo' => { '@id' => 'as:movedTo', '@type' => '@id' } },
+    also_known_as: { 'alsoKnownAs' => { '@id' => 'as:alsoKnownAs', '@type' => '@id' } },
+    emoji: { 'toot' => 'http://joinmastodon.org/ns#', 'Emoji' => 'toot:Emoji' },
+    featured: { 'toot' => 'http://joinmastodon.org/ns#', 'featured' => { '@id' => 'toot:featured', '@type' => '@id' }, 'featuredTags' => { '@id' => 'toot:featuredTags', '@type' => '@id' } },
+    property_value: { 'schema' => 'http://schema.org#', 'PropertyValue' => 'schema:PropertyValue', 'value' => 'schema:value' },
+    atom_uri: { 'ostatus' => 'http://ostatus.org#', 'atomUri' => 'ostatus:atomUri' },
+    conversation: { 'ostatus' => 'http://ostatus.org#', 'inReplyToAtomUri' => 'ostatus:inReplyToAtomUri', 'conversation' => 'ostatus:conversation' },
+    focal_point: { 'toot' => 'http://joinmastodon.org/ns#', 'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' } },
+    identity_proof: { 'toot' => 'http://joinmastodon.org/ns#', 'IdentityProof' => 'toot:IdentityProof' },
+    blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' },
+    discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' },
+    voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' },
+    olm: { 'toot' => 'http://joinmastodon.org/ns#', 'Device' => 'toot:Device', 'Ed25519Signature' => 'toot:Ed25519Signature', 'Ed25519Key' => 'toot:Ed25519Key', 'Curve25519Key' => 'toot:Curve25519Key', 'EncryptedMessage' => 'toot:EncryptedMessage', 'publicKeyBase64' => 'toot:publicKeyBase64', 'deviceId' => 'toot:deviceId', 'claim' => { '@type' => '@id', '@id' => 'toot:claim' }, 'fingerprintKey' => { '@type' => '@id', '@id' => 'toot:fingerprintKey' }, 'identityKey' => { '@type' => '@id', '@id' => 'toot:identityKey' }, 'devices' => { '@type' => '@id', '@id' => 'toot:devices' }, 'messageFranking' => 'toot:messageFranking', 'messageType' => 'toot:messageType', 'cipherText' => 'toot:cipherText' },
+    suspended: { 'toot' => 'http://joinmastodon.org/ns#', 'suspended' => 'toot:suspended' },
+  }.freeze
+
+  def full_context
+    serialized_context(NAMED_CONTEXT_MAP, CONTEXT_EXTENSION_MAP)
+  end
+
+  def serialized_context(named_contexts_map, context_extensions_map)
+    context_array = []
+
+    named_contexts     = named_contexts_map.keys
+    context_extensions = context_extensions_map.keys
+
+    named_contexts.each do |key|
+      context_array << NAMED_CONTEXT_MAP[key]
+    end
+
+    extensions = context_extensions.each_with_object({}) do |key, h|
+      h.merge!(CONTEXT_EXTENSION_MAP[key])
+    end
+
+    context_array << extensions unless extensions.empty?
+
+    if context_array.size == 1
+      context_array.first
+    else
+      context_array
+    end
+  end
+end

app/helpers/jsonld_helper.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module JsonLdHelper
+  include ContextHelper
+
   def equals_or_includes?(haystack, needle)
     haystack.is_a?(Array) ? haystack.include?(needle) : haystack == needle
   end
@@ -63,6 +65,84 @@ module JsonLdHelper
     graph.dump(:normalize)
   end
 
+  def compact(json)
+    compacted = JSON::LD::API.compact(json.without('signature'), full_context, documentLoader: method(:load_jsonld_context))
+    compacted['signature'] = json['signature']
+    compacted
+  end
+
+  # Patches a JSON-LD document to avoid compatibility issues on redistribution
+  #
+  # Since compacting a JSON-LD document against Mastodon's built-in vocabulary
+  # means other extension namespaces will be expanded, malformed JSON-LD
+  # attributes lost, and some values “unexpectedly” compacted this method
+  # patches the following likely sources of incompatibility:
+  # - 'https://www.w3.org/ns/activitystreams#Public' being compacted to
+  #   'as:Public' (for instance, pre-3.4.0 Mastodon does not understand
+  #   'as:Public')
+  # - single-item arrays being compacted to the item itself (`[foo]` being
+  #   compacted to `foo`)
+  #
+  # It is not always possible for `patch_for_forwarding!` to produce a document
+  # deemed safe for forwarding. Use `safe_for_forwarding?` to check the status
+  # of the output document.
+  #
+  # @param original [Hash] The original JSON-LD document used as reference
+  # @param compacted [Hash] The compacted JSON-LD document to be patched
+  # @return [void]
+  def patch_for_forwarding!(original, compacted)
+    original.without('@context', 'signature').each do |key, value|
+      next if value.nil? || !compacted.key?(key)
+
+      compacted_value = compacted[key]
+      if value.is_a?(Hash) && compacted_value.is_a?(Hash)
+        patch_for_forwarding!(value, compacted_value)
+      elsif value.is_a?(Array)
+        compacted_value = [compacted_value] unless compacted_value.is_a?(Array)
+        return if value.size != compacted_value.size
+
+        compacted[key] = value.zip(compacted_value).map do |v, vc|
+          if v.is_a?(Hash) && vc.is_a?(Hash)
+            patch_for_forwarding!(v, vc)
+            vc
+          elsif v == 'https://www.w3.org/ns/activitystreams#Public' && vc == 'as:Public'
+            v
+          else
+            vc
+          end
+        end
+      elsif value == 'https://www.w3.org/ns/activitystreams#Public' && compacted_value == 'as:Public'
+        compacted[key] = value
+      end
+    end
+  end
+
+  # Tests whether a JSON-LD compaction is deemed safe for redistribution,
+  # that is, if it doesn't change its meaning to consumers that do not actually
+  # handle JSON-LD, but rely on values being serialized in a certain way.
+  #
+  # See `patch_for_forwarding!` for details.
+  #
+  # @param original [Hash] The original JSON-LD document used as reference
+  # @param compacted [Hash] The compacted JSON-LD document to be patched
+  # @return [Boolean] Whether the patched document is deemed safe
+  def safe_for_forwarding?(original, compacted)
+    original.without('@context', 'signature').all? do |key, value|
+      compacted_value = compacted[key]
+      return false unless value.class == compacted_value.class
+
+      if value.is_a?(Hash)
+        safe_for_forwarding?(value, compacted_value)
+      elsif value.is_a?(Array)
+        value.zip(compacted_value).all? do |v, vc|
+          v.is_a?(Hash) ? (vc.is_a?(Hash) && safe_for_forwarding?(v, vc)) : v == vc
+        end
+      else
+        value == compacted_value
+      end
+    end
+  end
+
   def fetch_resource(uri, id, on_behalf_of = nil)
     unless id
       json = fetch_resource_without_id_validation(uri, on_behalf_of)

app/javascript/mastodon/actions/picture_in_picture.js

@@ -22,13 +22,20 @@ export const PICTURE_IN_PICTURE_REMOVE = 'PICTURE_IN_PICTURE_REMOVE';
  * @param {MediaProps} props
  * @return {object}
  */
-export const deployPictureInPicture = (statusId, accountId, playerType, props) => ({
-  type: PICTURE_IN_PICTURE_DEPLOY,
-  statusId,
-  accountId,
-  playerType,
-  props,
-});
+export const deployPictureInPicture = (statusId, accountId, playerType, props) => {
+  return (dispatch, getState) => {
+    // Do not open a player for a toot that does not exist
+    if (getState().hasIn(['statuses', statusId])) {
+      dispatch({
+        type: PICTURE_IN_PICTURE_DEPLOY,
+        statusId,
+        accountId,
+        playerType,
+        props,
+      });
+    }
+  };
+};
 
 /*
  * @return {object}

app/javascript/mastodon/components/intersection_observer_article.js

@@ -93,7 +93,7 @@ export default class IntersectionObserverArticle extends React.Component {
     // When the browser gets a chance, test if we're still not intersecting,
     // and if so, set our isHidden to true to trigger an unrender. The point of
     // this is to save DOM nodes and avoid using up too much memory.
-    // See: https://github.com/tootsuite/mastodon/issues/2900
+    // See: https://github.com/mastodon/mastodon/issues/2900
     this.setState((prevState) => ({ isHidden: !prevState.isIntersecting }));
   }
 

app/javascript/mastodon/components/modal_root.js

@@ -1,10 +1,15 @@
 import React from 'react';
 import PropTypes from 'prop-types';
 import 'wicg-inert';
+import { createBrowserHistory } from 'history';
 import { multiply } from 'color-blend';
 
 export default class ModalRoot extends React.PureComponent {
 
+  static contextTypes = {
+    router: PropTypes.object,
+  };
+
   static propTypes = {
     children: PropTypes.node,
     onClose: PropTypes.func.isRequired,
@@ -48,6 +53,7 @@ export default class ModalRoot extends React.PureComponent {
   componentDidMount () {
     window.addEventListener('keyup', this.handleKeyUp, false);
     window.addEventListener('keydown', this.handleKeyDown, false);
+    this.history = this.context.router ? this.context.router.history : createBrowserHistory();
   }
 
   componentWillReceiveProps (nextProps) {
@@ -69,6 +75,14 @@ export default class ModalRoot extends React.PureComponent {
         this.activeElement.focus({ preventScroll: true });
         this.activeElement = null;
       }).catch(console.error);
+
+      this._handleModalClose();
+    }
+    if (this.props.children && !prevProps.children) {
+      this._handleModalOpen();
+    }
+    if (this.props.children) {
+      this._ensureHistoryBuffer();
     }
   }
 
@@ -77,6 +91,32 @@ export default class ModalRoot extends React.PureComponent {
     window.removeEventListener('keydown', this.handleKeyDown);
   }
 
+  _handleModalOpen () {
+    this._modalHistoryKey = Date.now();
+    this.unlistenHistory = this.history.listen((_, action) => {
+      if (action === 'POP') {
+        this.props.onClose();
+      }
+    });
+  }
+
+  _handleModalClose () {
+    if (this.unlistenHistory) {
+      this.unlistenHistory();
+    }
+    const { state } = this.history.location;
+    if (state && state.mastodonModalKey === this._modalHistoryKey) {
+      this.history.goBack();
+    }
+  }
+
+  _ensureHistoryBuffer () {
+    const { pathname, state } = this.history.location;
+    if (!state || state.mastodonModalKey !== this._modalHistoryKey) {
+      this.history.push(pathname, { ...state, mastodonModalKey: this._modalHistoryKey });
+    }
+  }
+
   getSiblings = () => {
     return Array(...this.node.parentElement.childNodes).filter(node => node !== this.node);
   }

app/javascript/mastodon/components/scrollable_list.js

@@ -1,5 +1,5 @@
 import React, { PureComponent } from 'react';
-import { ScrollContainer } from 'react-router-scroll-4';
+import ScrollContainer from 'mastodon/containers/scroll_container';
 import PropTypes from 'prop-types';
 import IntersectionObserverArticleContainer from '../containers/intersection_observer_article_container';
 import LoadMore from './load_more';
@@ -34,7 +34,6 @@ class ScrollableList extends PureComponent {
     onScrollToTop: PropTypes.func,
     onScroll: PropTypes.func,
     trackScroll: PropTypes.bool,
-    shouldUpdateScroll: PropTypes.func,
     isLoading: PropTypes.bool,
     showLoading: PropTypes.bool,
     hasMore: PropTypes.bool,
@@ -290,7 +289,7 @@ class ScrollableList extends PureComponent {
   }
 
   render () {
-    const { children, scrollKey, trackScroll, shouldUpdateScroll, showLoading, isLoading, hasMore, numPending, prepend, alwaysPrepend, append, emptyMessage, onLoadMore } = this.props;
+    const { children, scrollKey, trackScroll, showLoading, isLoading, hasMore, numPending, prepend, alwaysPrepend, append, emptyMessage, onLoadMore } = this.props;
     const { fullscreen } = this.state;
     const childrenCount = React.Children.count(children);
 
@@ -356,7 +355,7 @@ class ScrollableList extends PureComponent {
 
     if (trackScroll) {
       return (
-        <ScrollContainer scrollKey={scrollKey} shouldUpdateScroll={shouldUpdateScroll}>
+        <ScrollContainer scrollKey={scrollKey}>
           {scrollableArea}
         </ScrollContainer>
       );

app/javascript/mastodon/components/status.js

@@ -309,8 +309,8 @@ class Status extends ImmutablePureComponent {
       return (
         <HotKeys handlers={handlers}>
           <div ref={this.handleRef} className={classNames('status__wrapper', { focusable: !this.props.muted })} tabIndex='0'>
-            {status.getIn(['account', 'display_name']) || status.getIn(['account', 'username'])}
-            {status.get('content')}
+            <span>{status.getIn(['account', 'display_name']) || status.getIn(['account', 'username'])}</span>
+            <span>{status.get('content')}</span>
           </div>
         </HotKeys>
       );

app/javascript/mastodon/components/status_list.js

@@ -18,7 +18,6 @@ export default class StatusList extends ImmutablePureComponent {
     onScrollToTop: PropTypes.func,
     onScroll: PropTypes.func,
     trackScroll: PropTypes.bool,
-    shouldUpdateScroll: PropTypes.func,
     isLoading: PropTypes.bool,
     isPartial: PropTypes.bool,
     hasMore: PropTypes.bool,
@@ -77,7 +76,7 @@ export default class StatusList extends ImmutablePureComponent {
   }
 
   render () {
-    const { statusIds, featuredStatusIds, shouldUpdateScroll, onLoadMore, timelineId, ...other }  = this.props;
+    const { statusIds, featuredStatusIds, onLoadMore, timelineId, ...other }  = this.props;
     const { isLoading, isPartial } = other;
 
     if (isPartial) {
@@ -120,7 +119,7 @@ export default class StatusList extends ImmutablePureComponent {
     }
 
     return (
-      <ScrollableList {...other} showLoading={isLoading && statusIds.size === 0} onLoadMore={onLoadMore && this.handleLoadOlder} shouldUpdateScroll={shouldUpdateScroll} ref={this.setRef}>
+      <ScrollableList {...other} showLoading={isLoading && statusIds.size === 0} onLoadMore={onLoadMore && this.handleLoadOlder} ref={this.setRef}>
         {scrollableContent}
       </ScrollableList>
     );

app/javascript/mastodon/containers/mastodon.js

@@ -10,8 +10,6 @@ import { hydrateStore } from '../actions/store';
 import { connectUserStream } from '../actions/streaming';
 import { IntlProvider, addLocaleData } from 'react-intl';
 import { getLocale } from '../locales';
-import { previewState as previewMediaState } from 'mastodon/features/ui/components/media_modal';
-import { previewState as previewVideoState } from 'mastodon/features/ui/components/video_modal';
 import initialState from '../initial_state';
 import ErrorBoundary from '../components/error_boundary';
 
@@ -41,8 +39,8 @@ export default class Mastodon extends React.PureComponent {
     }
   }
 
-  shouldUpdateScroll (_, { location }) {
-    return location.state !== previewMediaState && location.state !== previewVideoState;
+  shouldUpdateScroll (prevRouterProps, { location }) {
+    return !(location.state?.mastodonModalKey && location.state?.mastodonModalKey !== prevRouterProps?.location?.state?.mastodonModalKey);
   }
 
   render () {

app/javascript/mastodon/containers/scroll_container.js

@@ -0,0 +1,18 @@
+import { ScrollContainer as OriginalScrollContainer } from 'react-router-scroll-4';
+
+// ScrollContainer is used to automatically scroll to the top when pushing a
+// new history state and remembering the scroll position when going back.
+// There are a few things we need to do differently, though.
+const defaultShouldUpdateScroll = (prevRouterProps, { location }) => {
+  // If the change is caused by opening a modal, do not scroll to top
+  return !(location.state?.mastodonModalKey && location.state?.mastodonModalKey !== prevRouterProps?.location?.state?.mastodonModalKey);
+};
+
+export default
+class ScrollContainer extends OriginalScrollContainer {
+
+  static defaultProps = {
+    shouldUpdateScroll: defaultShouldUpdateScroll,
+  };
+
+}

app/javascript/mastodon/features/account_gallery/index.js

@@ -11,7 +11,7 @@ import ImmutablePureComponent from 'react-immutable-pure-component';
 import { getAccountGallery } from 'mastodon/selectors';
 import MediaItem from './components/media_item';
 import HeaderContainer from '../account_timeline/containers/header_container';
-import { ScrollContainer } from 'react-router-scroll-4';
+import ScrollContainer from 'mastodon/containers/scroll_container';
 import LoadMore from 'mastodon/components/load_more';
 import MissingIndicator from 'mastodon/components/missing_indicator';
 import { openModal } from 'mastodon/actions/modal';
@@ -29,7 +29,6 @@ const mapStateToProps = (state, props) => ({
 class LoadMoreMedia extends ImmutablePureComponent {
 
   static propTypes = {
-    shouldUpdateScroll: PropTypes.func,
     maxId: PropTypes.string,
     onLoadMore: PropTypes.func.isRequired,
   };
@@ -127,7 +126,7 @@ class AccountGallery extends ImmutablePureComponent {
   }
 
   render () {
-    const { attachments, shouldUpdateScroll, isLoading, hasMore, isAccount, multiColumn, blockedBy, suspended } = this.props;
+    const { attachments, isLoading, hasMore, isAccount, multiColumn, blockedBy, suspended } = this.props;
     const { width } = this.state;
 
     if (!isAccount) {
@@ -164,7 +163,7 @@ class AccountGallery extends ImmutablePureComponent {
       <Column>
         <ColumnBackButton multiColumn={multiColumn} />
 
-        <ScrollContainer scrollKey='account_gallery' shouldUpdateScroll={shouldUpdateScroll}>
+        <ScrollContainer scrollKey='account_gallery'>
           <div className='scrollable scrollable--flex' onScroll={this.handleScroll}>
             <HeaderContainer accountId={this.props.params.accountId} />
 

app/javascript/mastodon/features/account_timeline/index.js

@@ -50,7 +50,6 @@ class AccountTimeline extends ImmutablePureComponent {
   static propTypes = {
     params: PropTypes.object.isRequired,
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     statusIds: ImmutablePropTypes.list,
     featuredStatusIds: ImmutablePropTypes.list,
     isLoading: PropTypes.bool,
@@ -115,7 +114,7 @@ class AccountTimeline extends ImmutablePureComponent {
   }
 
   render () {
-    const { shouldUpdateScroll, statusIds, featuredStatusIds, isLoading, hasMore, blockedBy, suspended, isAccount, multiColumn, remote, remoteUrl } = this.props;
+    const { statusIds, featuredStatusIds, isLoading, hasMore, blockedBy, suspended, isAccount, multiColumn, remote, remoteUrl } = this.props;
 
     if (!isAccount) {
       return (
@@ -162,7 +161,6 @@ class AccountTimeline extends ImmutablePureComponent {
           isLoading={isLoading}
           hasMore={hasMore}
           onLoadMore={this.handleLoadMore}
-          shouldUpdateScroll={shouldUpdateScroll}
           emptyMessage={emptyMessage}
           bindToDocument={!multiColumn}
           timelineId='account'

app/javascript/mastodon/features/blocks/index.js

@@ -29,7 +29,6 @@ class Blocks extends ImmutablePureComponent {
   static propTypes = {
     params: PropTypes.object.isRequired,
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     accountIds: ImmutablePropTypes.list,
     hasMore: PropTypes.bool,
     isLoading: PropTypes.bool,
@@ -46,7 +45,7 @@ class Blocks extends ImmutablePureComponent {
   }, 300, { leading: true });
 
   render () {
-    const { intl, accountIds, shouldUpdateScroll, hasMore, multiColumn, isLoading } = this.props;
+    const { intl, accountIds, hasMore, multiColumn, isLoading } = this.props;
 
     if (!accountIds) {
       return (
@@ -66,7 +65,6 @@ class Blocks extends ImmutablePureComponent {
           onLoadMore={this.handleLoadMore}
           hasMore={hasMore}
           isLoading={isLoading}
-          shouldUpdateScroll={shouldUpdateScroll}
           emptyMessage={emptyMessage}
           bindToDocument={!multiColumn}
         >

app/javascript/mastodon/features/bookmarked_statuses/index.js

@@ -27,7 +27,6 @@ class Bookmarks extends ImmutablePureComponent {
 
   static propTypes = {
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     statusIds: ImmutablePropTypes.list.isRequired,
     intl: PropTypes.object.isRequired,
     columnId: PropTypes.string,
@@ -68,7 +67,7 @@ class Bookmarks extends ImmutablePureComponent {
   }, 300, { leading: true })
 
   render () {
-    const { intl, shouldUpdateScroll, statusIds, columnId, multiColumn, hasMore, isLoading } = this.props;
+    const { intl, statusIds, columnId, multiColumn, hasMore, isLoading } = this.props;
     const pinned = !!columnId;
 
     const emptyMessage = <FormattedMessage id='empty_column.bookmarked_statuses' defaultMessage="You don't have any bookmarked toots yet. When you bookmark one, it will show up here." />;
@@ -93,7 +92,6 @@ class Bookmarks extends ImmutablePureComponent {
           hasMore={hasMore}
           isLoading={isLoading}
           onLoadMore={this.handleLoadMore}
-          shouldUpdateScroll={shouldUpdateScroll}
           emptyMessage={emptyMessage}
           bindToDocument={!multiColumn}
         />

app/javascript/mastodon/features/community_timeline/index.js

@@ -41,7 +41,6 @@ class CommunityTimeline extends React.PureComponent {
 
   static propTypes = {
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     columnId: PropTypes.string,
     intl: PropTypes.object.isRequired,
     hasUnread: PropTypes.bool,
@@ -103,7 +102,7 @@ class CommunityTimeline extends React.PureComponent {
   }
 
   render () {
-    const { intl, shouldUpdateScroll, hasUnread, columnId, multiColumn, onlyMedia } = this.props;
+    const { intl, hasUnread, columnId, multiColumn, onlyMedia } = this.props;
     const pinned = !!columnId;
 
     return (
@@ -127,7 +126,6 @@ class CommunityTimeline extends React.PureComponent {
           timelineId={`community${onlyMedia ? ':media' : ''}`}
           onLoadMore={this.handleLoadMore}
           emptyMessage={<FormattedMessage id='empty_column.community' defaultMessage='The local timeline is empty. Write something publicly to get the ball rolling!' />}
-          shouldUpdateScroll={shouldUpdateScroll}
           bindToDocument={!multiColumn}
         />
       </Column>

app/javascript/mastodon/features/compose/containers/navigation_container.js

@@ -21,6 +21,7 @@ const mapDispatchToProps = (dispatch, { intl }) => ({
     dispatch(openModal('CONFIRM', {
       message: intl.formatMessage(messages.logoutMessage),
       confirm: intl.formatMessage(messages.logoutConfirm),
+      closeWhenConfirm: false,
       onConfirm: () => logOut(),
     }));
   },

app/javascript/mastodon/features/compose/index.js

@@ -74,6 +74,7 @@ class Compose extends React.PureComponent {
     dispatch(openModal('CONFIRM', {
       message: intl.formatMessage(messages.logoutMessage),
       confirm: intl.formatMessage(messages.logoutConfirm),
+      closeWhenConfirm: false,
       onConfirm: () => logOut(),
     }));
 

app/javascript/mastodon/features/direct_timeline/components/conversations_list.js

@@ -14,7 +14,6 @@ export default class ConversationsList extends ImmutablePureComponent {
     hasMore: PropTypes.bool,
     isLoading: PropTypes.bool,
     onLoadMore: PropTypes.func,
-    shouldUpdateScroll: PropTypes.func,
   };
 
   getCurrentIndex = id => this.props.conversations.findIndex(x => x.get('id') === id)

app/javascript/mastodon/features/direct_timeline/index.js

@@ -19,7 +19,6 @@ class DirectTimeline extends React.PureComponent {
 
   static propTypes = {
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     columnId: PropTypes.string,
     intl: PropTypes.object.isRequired,
     hasUnread: PropTypes.bool,
@@ -71,7 +70,7 @@ class DirectTimeline extends React.PureComponent {
   }
 
   render () {
-    const { intl, hasUnread, columnId, multiColumn, shouldUpdateScroll } = this.props;
+    const { intl, hasUnread, columnId, multiColumn } = this.props;
     const pinned = !!columnId;
 
     return (
@@ -93,7 +92,6 @@ class DirectTimeline extends React.PureComponent {
           timelineId='direct'
           onLoadMore={this.handleLoadMore}
           emptyMessage={<FormattedMessage id='empty_column.direct' defaultMessage="You don't have any direct messages yet. When you send or receive one, it will show up here." />}
-          shouldUpdateScroll={shouldUpdateScroll}
         />
       </Column>
     );

app/javascript/mastodon/features/directory/index.js

@@ -12,7 +12,7 @@ import AccountCard from './components/account_card';
 import RadioButton from 'mastodon/components/radio_button';
 import classNames from 'classnames';
 import LoadMore from 'mastodon/components/load_more';
-import { ScrollContainer } from 'react-router-scroll-4';
+import ScrollContainer from 'mastodon/containers/scroll_container';
 
 const messages = defineMessages({
   title: { id: 'column.directory', defaultMessage: 'Browse profiles' },
@@ -40,7 +40,6 @@ class Directory extends React.PureComponent {
     isLoading: PropTypes.bool,
     accountIds: ImmutablePropTypes.list.isRequired,
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     columnId: PropTypes.string,
     intl: PropTypes.object.isRequired,
     multiColumn: PropTypes.bool,
@@ -125,7 +124,7 @@ class Directory extends React.PureComponent {
   }
 
   render () {
-    const { isLoading, accountIds, intl, columnId, multiColumn, domain, shouldUpdateScroll } = this.props;
+    const { isLoading, accountIds, intl, columnId, multiColumn, domain } = this.props;
     const { order, local }  = this.getParams(this.props, this.state);
     const pinned = !!columnId;
 
@@ -163,7 +162,7 @@ class Directory extends React.PureComponent {
           multiColumn={multiColumn}
         />
 
-        {multiColumn && !pinned ? <ScrollContainer scrollKey='directory' shouldUpdateScroll={shouldUpdateScroll}>{scrollableArea}</ScrollContainer> : scrollableArea}
+        {multiColumn && !pinned ? <ScrollContainer scrollKey='directory'>{scrollableArea}</ScrollContainer> : scrollableArea}
       </Column>
     );
   }

app/javascript/mastodon/features/domain_blocks/index.js

@@ -29,7 +29,6 @@ class Blocks extends ImmutablePureComponent {
   static propTypes = {
     params: PropTypes.object.isRequired,
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     hasMore: PropTypes.bool,
     domains: ImmutablePropTypes.orderedSet,
     intl: PropTypes.object.isRequired,
@@ -45,7 +44,7 @@ class Blocks extends ImmutablePureComponent {
   }, 300, { leading: true });
 
   render () {
-    const { intl, domains, shouldUpdateScroll, hasMore, multiColumn } = this.props;
+    const { intl, domains, hasMore, multiColumn } = this.props;
 
     if (!domains) {
       return (
@@ -64,7 +63,6 @@ class Blocks extends ImmutablePureComponent {
           scrollKey='domain_blocks'
           onLoadMore={this.handleLoadMore}
           hasMore={hasMore}
-          shouldUpdateScroll={shouldUpdateScroll}
           emptyMessage={emptyMessage}
           bindToDocument={!multiColumn}
         >

app/javascript/mastodon/features/emoji/__tests__/emoji-test.js

@@ -11,8 +11,8 @@ describe('emoji', () => {
     });
 
     it('works with unclosed tags', () => {
-      expect(emojify('hello>')).toEqual('hello>');
-      expect(emojify('<hello')).toEqual('<hello');
+      expect(emojify('hello>')).toEqual('hello&gt;');
+      expect(emojify('<hello')).toEqual('');
     });
 
     it('works with unclosed shortcodes', () => {
@@ -22,23 +22,23 @@ describe('emoji', () => {
 
     it('does unicode', () => {
       expect(emojify('\uD83D\uDC69\u200D\uD83D\uDC69\u200D\uD83D\uDC66\u200D\uD83D\uDC66')).toEqual(
-        '<img draggable="false" class="emojione" alt="👩‍👩‍👦‍👦" title=":woman-woman-boy-boy:" src="/emoji/1f469-200d-1f469-200d-1f466-200d-1f466.svg" />');
+        '<img draggable="false" class="emojione" alt="👩‍👩‍👦‍👦" title=":woman-woman-boy-boy:" src="/emoji/1f469-200d-1f469-200d-1f466-200d-1f466.svg">');
       expect(emojify('👨‍👩‍👧‍👧')).toEqual(
-        '<img draggable="false" class="emojione" alt="👨‍👩‍👧‍👧" title=":man-woman-girl-girl:" src="/emoji/1f468-200d-1f469-200d-1f467-200d-1f467.svg" />');
-      expect(emojify('👩‍👩‍👦')).toEqual('<img draggable="false" class="emojione" alt="👩‍👩‍👦" title=":woman-woman-boy:" src="/emoji/1f469-200d-1f469-200d-1f466.svg" />');
+        '<img draggable="false" class="emojione" alt="👨‍👩‍👧‍👧" title=":man-woman-girl-girl:" src="/emoji/1f468-200d-1f469-200d-1f467-200d-1f467.svg">');
+      expect(emojify('👩‍👩‍👦')).toEqual('<img draggable="false" class="emojione" alt="👩‍👩‍👦" title=":woman-woman-boy:" src="/emoji/1f469-200d-1f469-200d-1f466.svg">');
       expect(emojify('\u2757')).toEqual(
-        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" />');
+        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg">');
     });
 
     it('does multiple unicode', () => {
       expect(emojify('\u2757 #\uFE0F\u20E3')).toEqual(
-        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" /> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg" />');
+        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg">');
       expect(emojify('\u2757#\uFE0F\u20E3')).toEqual(
-        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" /><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg" />');
+        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg">');
       expect(emojify('\u2757 #\uFE0F\u20E3 \u2757')).toEqual(
-        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" /> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg" /> <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" />');
+        '<img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"> <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg">');
       expect(emojify('foo \u2757 #\uFE0F\u20E3 bar')).toEqual(
-        'foo <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" /> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg" /> bar');
+        'foo <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg"> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/23-20e3.svg"> bar');
     });
 
     it('ignores unicode inside of tags', () => {
@@ -46,16 +46,16 @@ describe('emoji', () => {
     });
 
     it('does multiple emoji properly (issue 5188)', () => {
-      expect(emojify('👌🌈💕')).toEqual('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg" /><img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg" /><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg" />');
-      expect(emojify('👌 🌈 💕')).toEqual('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg" /> <img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg" /> <img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg" />');
+      expect(emojify('👌🌈💕')).toEqual('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg"><img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg"><img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg">');
+      expect(emojify('👌 🌈 💕')).toEqual('<img draggable="false" class="emojione" alt="👌" title=":ok_hand:" src="/emoji/1f44c.svg"> <img draggable="false" class="emojione" alt="🌈" title=":rainbow:" src="/emoji/1f308.svg"> <img draggable="false" class="emojione" alt="💕" title=":two_hearts:" src="/emoji/1f495.svg">');
     });
 
     it('does an emoji that has no shortcode', () => {
-      expect(emojify('👁‍🗨')).toEqual('<img draggable="false" class="emojione" alt="👁‍🗨" title="" src="/emoji/1f441-200d-1f5e8.svg" />');
+      expect(emojify('👁‍🗨')).toEqual('<img draggable="false" class="emojione" alt="👁‍🗨" title="" src="/emoji/1f441-200d-1f5e8.svg">');
     });
 
     it('does an emoji whose filename is irregular', () => {
-      expect(emojify('↙️')).toEqual('<img draggable="false" class="emojione" alt="↙️" title=":arrow_lower_left:" src="/emoji/2199.svg" />');
+      expect(emojify('↙️')).toEqual('<img draggable="false" class="emojione" alt="↙️" title=":arrow_lower_left:" src="/emoji/2199.svg">');
     });
 
     it('avoid emojifying on invisible text', () => {
@@ -67,26 +67,26 @@ describe('emoji', () => {
 
     it('avoid emojifying on invisible text with nested tags', () => {
       expect(emojify('<span class="invisible">😄<span class="foo">bar</span>😴</span>😇'))
-        .toEqual('<span class="invisible">😄<span class="foo">bar</span>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg" />');
+        .toEqual('<span class="invisible">😄<span class="foo">bar</span>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg">');
       expect(emojify('<span class="invisible">😄<span class="invisible">😕</span>😴</span>😇'))
-        .toEqual('<span class="invisible">😄<span class="invisible">😕</span>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg" />');
-      expect(emojify('<span class="invisible">😄<br/>😴</span>😇'))
-        .toEqual('<span class="invisible">😄<br/>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg" />');
+        .toEqual('<span class="invisible">😄<span class="invisible">😕</span>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg">');
+      expect(emojify('<span class="invisible">😄<br>😴</span>😇'))
+        .toEqual('<span class="invisible">😄<br>😴</span><img draggable="false" class="emojione" alt="😇" title=":innocent:" src="/emoji/1f607.svg">');
     });
 
     it('skips the textual presentation VS15 character', () => {
       expect(emojify('✴︎')) // This is U+2734 EIGHT POINTED BLACK STAR then U+FE0E VARIATION SELECTOR-15
-        .toEqual('<img draggable="false" class="emojione" alt="✴" title=":eight_pointed_black_star:" src="/emoji/2734_border.svg" />');
+        .toEqual('<img draggable="false" class="emojione" alt="✴" title=":eight_pointed_black_star:" src="/emoji/2734_border.svg">');
     });
 
     it('does an simple emoji properly', () => {
       expect(emojify('♀♂'))
-        .toEqual('<img draggable="false" class="emojione" alt="♀" title=":female_sign:" src="/emoji/2640.svg" /><img draggable="false" class="emojione" alt="♂" title=":male_sign:" src="/emoji/2642.svg" />');
+        .toEqual('<img draggable="false" class="emojione" alt="♀" title=":female_sign:" src="/emoji/2640.svg"><img draggable="false" class="emojione" alt="♂" title=":male_sign:" src="/emoji/2642.svg">');
     });
 
     it('does an emoji containing ZWJ properly', () => {
       expect(emojify('💂‍♀️💂‍♂️'))
-        .toEqual('<img draggable="false" class="emojione" alt="💂\u200D♀️" title=":female-guard:" src="/emoji/1f482-200d-2640-fe0f_border.svg" /><img draggable="false" class="emojione" alt="💂\u200D♂️" title=":male-guard:" src="/emoji/1f482-200d-2642-fe0f_border.svg" />');
+        .toEqual('<img draggable="false" class="emojione" alt="💂\u200D♀️" title=":female-guard:" src="/emoji/1f482-200d-2640-fe0f_border.svg"><img draggable="false" class="emojione" alt="💂\u200D♂️" title=":male-guard:" src="/emoji/1f482-200d-2642-fe0f_border.svg">');
     });
   });
 });

app/javascript/mastodon/features/emoji/emoji.js

@@ -19,15 +19,26 @@ const emojiFilename = (filename) => {
   return borderedEmoji.includes(filename) ? (filename + '_border') : filename;
 };
 
-const emojify = (str, customEmojis = {}) => {
-  const tagCharsWithoutEmojis = '<&';
-  const tagCharsWithEmojis = Object.keys(customEmojis).length ? '<&:' : '<&';
-  let rtn = '', tagChars = tagCharsWithEmojis, invisible = 0;
+const domParser = new DOMParser();
+
+const emojifyTextNode = (node, customEmojis) => {
+  let str = node.textContent;
+
+  const fragment = new DocumentFragment();
+
   for (;;) {
-    let match, i = 0, tag;
-    while (i < str.length && (tag = tagChars.indexOf(str[i])) === -1 && (invisible || !(match = trie.search(str.slice(i))))) {
-      i += str.codePointAt(i) < 65536 ? 1 : 2;
+    let match, i = 0;
+
+    if (customEmojis === null) {
+      while (i < str.length && !(match = trie.search(str.slice(i)))) {
+        i += str.codePointAt(i) < 65536 ? 1 : 2;
+      }
+    } else {
+      while (i < str.length && str[i] !== ':' && !(match = trie.search(str.slice(i)))) {
+        i += str.codePointAt(i) < 65536 ? 1 : 2;
+      }
     }
+
     let rend, replacement = '';
     if (i === str.length) {
       break;
@@ -35,8 +46,6 @@ const emojify = (str, customEmojis = {}) => {
       if (!(() => {
         rend = str.indexOf(':', i + 1) + 1;
         if (!rend) return false; // no pair of ':'
-        const lt = str.indexOf('<', i + 1);
-        if (!(lt === -1 || lt >= rend)) return false; // tag appeared before closing ':'
         const shortname = str.slice(i, rend);
         // now got a replacee as ':shortname:'
         // if you want additional emoji handler, add statements below which set replacement and return true.
@@ -47,29 +56,6 @@ const emojify = (str, customEmojis = {}) => {
         }
         return false;
       })()) rend = ++i;
-    } else if (tag >= 0) { // <, &
-      rend = str.indexOf('>;'[tag], i + 1) + 1;
-      if (!rend) {
-        break;
-      }
-      if (tag === 0) {
-        if (invisible) {
-          if (str[i + 1] === '/') { // closing tag
-            if (!--invisible) {
-              tagChars = tagCharsWithEmojis;
-            }
-          } else if (str[rend - 2] !== '/') { // opening tag
-            invisible++;
-          }
-        } else {
-          if (str.startsWith('<span class="invisible">', i)) {
-            // avoid emojifying on invisible text
-            invisible = 1;
-            tagChars = tagCharsWithoutEmojis;
-          }
-        }
-      }
-      i = rend;
     } else { // matched to unicode emoji
       const { filename, shortCode } = unicodeMapping[match];
       const title = shortCode ? `:${shortCode}:` : '';
@@ -80,10 +66,43 @@ const emojify = (str, customEmojis = {}) => {
         rend += 1;
       }
     }
-    rtn += str.slice(0, i) + replacement;
+
+    fragment.append(document.createTextNode(str.slice(0, i)));
+    if (replacement) {
+      fragment.append(domParser.parseFromString(replacement, 'text/html').documentElement.getElementsByTagName('img')[0]);
+    }
+    node.textContent = str.slice(0, i);
     str = str.slice(rend);
   }
-  return rtn + str;
+
+  fragment.append(document.createTextNode(str));
+  node.parentElement.replaceChild(fragment, node);
+};
+
+const emojifyNode = (node, customEmojis) => {
+  for (const child of node.childNodes) {
+    switch(child.nodeType) {
+    case Node.TEXT_NODE:
+      emojifyTextNode(child, customEmojis);
+      break;
+    case Node.ELEMENT_NODE:
+      if (!child.classList.contains('invisible'))
+        emojifyNode(child, customEmojis);
+      break;
+    }
+  }
+};
+
+const emojify = (str, customEmojis = {}) => {
+  const wrapper = document.createElement('div');
+  wrapper.innerHTML = str;
+
+  if (!Object.keys(customEmojis).length)
+    customEmojis = null;
+
+  emojifyNode(wrapper, customEmojis);
+
+  return wrapper.innerHTML;
 };
 
 export default emojify;

app/javascript/mastodon/features/favourited_statuses/index.js

@@ -27,7 +27,6 @@ class Favourites extends ImmutablePureComponent {
 
   static propTypes = {
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     statusIds: ImmutablePropTypes.list.isRequired,
     intl: PropTypes.object.isRequired,
     columnId: PropTypes.string,
@@ -68,7 +67,7 @@ class Favourites extends ImmutablePureComponent {
   }, 300, { leading: true })
 
   render () {
-    const { intl, shouldUpdateScroll, statusIds, columnId, multiColumn, hasMore, isLoading } = this.props;
+    const { intl, statusIds, columnId, multiColumn, hasMore, isLoading } = this.props;
     const pinned = !!columnId;
 
     const emptyMessage = <FormattedMessage id='empty_column.favourited_statuses' defaultMessage="You don't have any favourite toots yet. When you favourite one, it will show up here." />;
@@ -93,7 +92,6 @@ class Favourites extends ImmutablePureComponent {
           hasMore={hasMore}
           isLoading={isLoading}
           onLoadMore={this.handleLoadMore}
-          shouldUpdateScroll={shouldUpdateScroll}
           emptyMessage={emptyMessage}
           bindToDocument={!multiColumn}
         />

app/javascript/mastodon/features/favourites/index.js

@@ -27,7 +27,6 @@ class Favourites extends ImmutablePureComponent {
   static propTypes = {
     params: PropTypes.object.isRequired,
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     accountIds: ImmutablePropTypes.list,
     multiColumn: PropTypes.bool,
     intl: PropTypes.object.isRequired,
@@ -50,7 +49,7 @@ class Favourites extends ImmutablePureComponent {
   }
 
   render () {
-    const { intl, shouldUpdateScroll, accountIds, multiColumn } = this.props;
+    const { intl, accountIds, multiColumn } = this.props;
 
     if (!accountIds) {
       return (
@@ -74,7 +73,6 @@ class Favourites extends ImmutablePureComponent {
 
         <ScrollableList
           scrollKey='favourites'
-          shouldUpdateScroll={shouldUpdateScroll}
           emptyMessage={emptyMessage}
           bindToDocument={!multiColumn}
         >

app/javascript/mastodon/features/follow_requests/index.js

@@ -32,7 +32,6 @@ class FollowRequests extends ImmutablePureComponent {
   static propTypes = {
     params: PropTypes.object.isRequired,
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     hasMore: PropTypes.bool,
     isLoading: PropTypes.bool,
     accountIds: ImmutablePropTypes.list,
@@ -51,7 +50,7 @@ class FollowRequests extends ImmutablePureComponent {
   }, 300, { leading: true });
 
   render () {
-    const { intl, shouldUpdateScroll, accountIds, hasMore, multiColumn, locked, domain, isLoading } = this.props;
+    const { intl, accountIds, hasMore, multiColumn, locked, domain, isLoading } = this.props;
 
     if (!accountIds) {
       return (
@@ -80,7 +79,6 @@ class FollowRequests extends ImmutablePureComponent {
           onLoadMore={this.handleLoadMore}
           hasMore={hasMore}
           isLoading={isLoading}
-          shouldUpdateScroll={shouldUpdateScroll}
           emptyMessage={emptyMessage}
           bindToDocument={!multiColumn}
           prepend={unlockedPrependMessage}

app/javascript/mastodon/features/followers/index.js

@@ -43,7 +43,6 @@ class Followers extends ImmutablePureComponent {
   static propTypes = {
     params: PropTypes.object.isRequired,
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     accountIds: ImmutablePropTypes.list,
     hasMore: PropTypes.bool,
     isLoading: PropTypes.bool,
@@ -73,7 +72,7 @@ class Followers extends ImmutablePureComponent {
   }, 300, { leading: true });
 
   render () {
-    const { shouldUpdateScroll, accountIds, hasMore, blockedBy, isAccount, multiColumn, isLoading, remote, remoteUrl } = this.props;
+    const { accountIds, hasMore, blockedBy, isAccount, multiColumn, isLoading, remote, remoteUrl } = this.props;
 
     if (!isAccount) {
       return (
@@ -112,7 +111,6 @@ class Followers extends ImmutablePureComponent {
           hasMore={hasMore}
           isLoading={isLoading}
           onLoadMore={this.handleLoadMore}
-          shouldUpdateScroll={shouldUpdateScroll}
           prepend={<HeaderContainer accountId={this.props.params.accountId} hideTabs />}
           alwaysPrepend
           append={remoteMessage}

app/javascript/mastodon/features/following/index.js

@@ -43,7 +43,6 @@ class Following extends ImmutablePureComponent {
   static propTypes = {
     params: PropTypes.object.isRequired,
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     accountIds: ImmutablePropTypes.list,
     hasMore: PropTypes.bool,
     isLoading: PropTypes.bool,
@@ -73,7 +72,7 @@ class Following extends ImmutablePureComponent {
   }, 300, { leading: true });
 
   render () {
-    const { shouldUpdateScroll, accountIds, hasMore, blockedBy, isAccount, multiColumn, isLoading, remote, remoteUrl } = this.props;
+    const { accountIds, hasMore, blockedBy, isAccount, multiColumn, isLoading, remote, remoteUrl } = this.props;
 
     if (!isAccount) {
       return (
@@ -112,7 +111,6 @@ class Following extends ImmutablePureComponent {
           hasMore={hasMore}
           isLoading={isLoading}
           onLoadMore={this.handleLoadMore}
-          shouldUpdateScroll={shouldUpdateScroll}
           prepend={<HeaderContainer accountId={this.props.params.accountId} hideTabs />}
           alwaysPrepend
           append={remoteMessage}

app/javascript/mastodon/features/hashtag_timeline/index.js

@@ -24,7 +24,6 @@ class HashtagTimeline extends React.PureComponent {
     params: PropTypes.object.isRequired,
     columnId: PropTypes.string,
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     hasUnread: PropTypes.bool,
     multiColumn: PropTypes.bool,
   };
@@ -130,7 +129,7 @@ class HashtagTimeline extends React.PureComponent {
   }
 
   render () {
-    const { shouldUpdateScroll, hasUnread, columnId, multiColumn } = this.props;
+    const { hasUnread, columnId, multiColumn } = this.props;
     const { id, local } = this.props.params;
     const pinned = !!columnId;
 
@@ -156,7 +155,6 @@ class HashtagTimeline extends React.PureComponent {
           timelineId={`hashtag:${id}${local ? ':local' : ''}`}
           onLoadMore={this.handleLoadMore}
           emptyMessage={<FormattedMessage id='empty_column.hashtag' defaultMessage='There is nothing in this hashtag yet.' />}
-          shouldUpdateScroll={shouldUpdateScroll}
           bindToDocument={!multiColumn}
         />
       </Column>

app/javascript/mastodon/features/home_timeline/index.js

@@ -34,7 +34,6 @@ class HomeTimeline extends React.PureComponent {
 
   static propTypes = {
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     intl: PropTypes.object.isRequired,
     hasUnread: PropTypes.bool,
     isPartial: PropTypes.bool,
@@ -112,7 +111,7 @@ class HomeTimeline extends React.PureComponent {
   }
 
   render () {
-    const { intl, shouldUpdateScroll, hasUnread, columnId, multiColumn, hasAnnouncements, unreadAnnouncements, showAnnouncements } = this.props;
+    const { intl, hasUnread, columnId, multiColumn, hasAnnouncements, unreadAnnouncements, showAnnouncements } = this.props;
     const pinned = !!columnId;
 
     let announcementsButton = null;
@@ -154,7 +153,6 @@ class HomeTimeline extends React.PureComponent {
           onLoadMore={this.handleLoadMore}
           timelineId='home'
           emptyMessage={<FormattedMessage id='empty_column.home' defaultMessage='Your home timeline is empty! Follow more people to fill it up. {suggestions}' values={{ suggestions: <Link to='/start'><FormattedMessage id='empty_column.home.suggestions' defaultMessage='See some suggestions' /></Link> }} />}
-          shouldUpdateScroll={shouldUpdateScroll}
           bindToDocument={!multiColumn}
         />
       </Column>

app/javascript/mastodon/features/list_timeline/index.js

@@ -41,7 +41,6 @@ class ListTimeline extends React.PureComponent {
   static propTypes = {
     params: PropTypes.object.isRequired,
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     columnId: PropTypes.string,
     hasUnread: PropTypes.bool,
     multiColumn: PropTypes.bool,
@@ -142,7 +141,7 @@ class ListTimeline extends React.PureComponent {
   }
 
   render () {
-    const { shouldUpdateScroll, hasUnread, columnId, multiColumn, list, intl } = this.props;
+    const { hasUnread, columnId, multiColumn, list, intl } = this.props;
     const { id } = this.props.params;
     const pinned = !!columnId;
     const title  = list ? list.get('title') : id;
@@ -207,7 +206,6 @@ class ListTimeline extends React.PureComponent {
           timelineId={`list:${id}`}
           onLoadMore={this.handleLoadMore}
           emptyMessage={<FormattedMessage id='empty_column.list' defaultMessage='There is nothing in this list yet. When members of this list post new statuses, they will appear here.' />}
-          shouldUpdateScroll={shouldUpdateScroll}
           bindToDocument={!multiColumn}
         />
       </Column>

app/javascript/mastodon/features/lists/index.js

@@ -48,7 +48,7 @@ class Lists extends ImmutablePureComponent {
   }
 
   render () {
-    const { intl, shouldUpdateScroll, lists, multiColumn } = this.props;
+    const { intl, lists, multiColumn } = this.props;
 
     if (!lists) {
       return (
@@ -68,7 +68,6 @@ class Lists extends ImmutablePureComponent {
 
         <ScrollableList
           scrollKey='lists'
-          shouldUpdateScroll={shouldUpdateScroll}
           emptyMessage={emptyMessage}
           prepend={<ColumnSubheading text={intl.formatMessage(messages.subheading)} />}
           bindToDocument={!multiColumn}

app/javascript/mastodon/features/mutes/index.js

@@ -29,7 +29,6 @@ class Mutes extends ImmutablePureComponent {
   static propTypes = {
     params: PropTypes.object.isRequired,
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     hasMore: PropTypes.bool,
     isLoading: PropTypes.bool,
     accountIds: ImmutablePropTypes.list,
@@ -46,7 +45,7 @@ class Mutes extends ImmutablePureComponent {
   }, 300, { leading: true });
 
   render () {
-    const { intl, shouldUpdateScroll, hasMore, accountIds, multiColumn, isLoading } = this.props;
+    const { intl, hasMore, accountIds, multiColumn, isLoading } = this.props;
 
     if (!accountIds) {
       return (
@@ -66,7 +65,6 @@ class Mutes extends ImmutablePureComponent {
           onLoadMore={this.handleLoadMore}
           hasMore={hasMore}
           isLoading={isLoading}
-          shouldUpdateScroll={shouldUpdateScroll}
           emptyMessage={emptyMessage}
           bindToDocument={!multiColumn}
         >

app/javascript/mastodon/features/notifications/index.js

@@ -74,7 +74,6 @@ class Notifications extends React.PureComponent {
     notifications: ImmutablePropTypes.list.isRequired,
     showFilterBar: PropTypes.bool.isRequired,
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     intl: PropTypes.object.isRequired,
     isLoading: PropTypes.bool,
     isUnread: PropTypes.bool,
@@ -176,7 +175,7 @@ class Notifications extends React.PureComponent {
   };
 
   render () {
-    const { intl, notifications, shouldUpdateScroll, isLoading, isUnread, columnId, multiColumn, hasMore, numPending, showFilterBar, lastReadId, canMarkAsRead, needsNotificationPermission } = this.props;
+    const { intl, notifications, isLoading, isUnread, columnId, multiColumn, hasMore, numPending, showFilterBar, lastReadId, canMarkAsRead, needsNotificationPermission } = this.props;
     const pinned = !!columnId;
     const emptyMessage = <FormattedMessage id='empty_column.notifications' defaultMessage="You don't have any notifications yet. When other people interact with you, you will see it here." />;
 
@@ -227,7 +226,6 @@ class Notifications extends React.PureComponent {
         onLoadPending={this.handleLoadPending}
         onScrollToTop={this.handleScrollToTop}
         onScroll={this.handleScroll}
-        shouldUpdateScroll={shouldUpdateScroll}
         bindToDocument={!multiColumn}
       >
         {scrollableContent}

app/javascript/mastodon/features/picture_in_picture/components/footer.js

@@ -114,7 +114,11 @@ class Footer extends ImmutablePureComponent {
       return;
     }
 
-    const { status } = this.props;
+    const { status, onClose } = this.props;
+
+    if (onClose) {
+      onClose();
+    }
 
     router.history.push(`/statuses/${status.get('id')}`);
   }

app/javascript/mastodon/features/pinned_statuses/index.js

@@ -24,7 +24,6 @@ class PinnedStatuses extends ImmutablePureComponent {
 
   static propTypes = {
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     statusIds: ImmutablePropTypes.list.isRequired,
     intl: PropTypes.object.isRequired,
     hasMore: PropTypes.bool.isRequired,
@@ -44,7 +43,7 @@ class PinnedStatuses extends ImmutablePureComponent {
   }
 
   render () {
-    const { intl, shouldUpdateScroll, statusIds, hasMore, multiColumn } = this.props;
+    const { intl, statusIds, hasMore, multiColumn } = this.props;
 
     return (
       <Column bindToDocument={!multiColumn} icon='thumb-tack' heading={intl.formatMessage(messages.heading)} ref={this.setRef}>
@@ -53,7 +52,6 @@ class PinnedStatuses extends ImmutablePureComponent {
           statusIds={statusIds}
           scrollKey='pinned_statuses'
           hasMore={hasMore}
-          shouldUpdateScroll={shouldUpdateScroll}
           bindToDocument={!multiColumn}
         />
       </Column>

app/javascript/mastodon/features/public_timeline/index.js

@@ -43,7 +43,6 @@ class PublicTimeline extends React.PureComponent {
 
   static propTypes = {
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     intl: PropTypes.object.isRequired,
     columnId: PropTypes.string,
     multiColumn: PropTypes.bool,
@@ -106,7 +105,7 @@ class PublicTimeline extends React.PureComponent {
   }
 
   render () {
-    const { intl, shouldUpdateScroll, columnId, hasUnread, multiColumn, onlyMedia, onlyRemote } = this.props;
+    const { intl, columnId, hasUnread, multiColumn, onlyMedia, onlyRemote } = this.props;
     const pinned = !!columnId;
 
     return (
@@ -130,7 +129,6 @@ class PublicTimeline extends React.PureComponent {
           trackScroll={!pinned}
           scrollKey={`public_timeline-${columnId}`}
           emptyMessage={<FormattedMessage id='empty_column.public' defaultMessage='There is nothing here! Write something publicly, or manually follow users from other servers to fill it up' />}
-          shouldUpdateScroll={shouldUpdateScroll}
           bindToDocument={!multiColumn}
         />
       </Column>

app/javascript/mastodon/features/reblogs/index.js

@@ -27,7 +27,6 @@ class Reblogs extends ImmutablePureComponent {
   static propTypes = {
     params: PropTypes.object.isRequired,
     dispatch: PropTypes.func.isRequired,
-    shouldUpdateScroll: PropTypes.func,
     accountIds: ImmutablePropTypes.list,
     multiColumn: PropTypes.bool,
     intl: PropTypes.object.isRequired,
@@ -50,7 +49,7 @@ class Reblogs extends ImmutablePureComponent {
   }
 
   render () {
-    const { intl, shouldUpdateScroll, accountIds, multiColumn } = this.props;
+    const { intl, accountIds, multiColumn } = this.props;
 
     if (!accountIds) {
       return (
@@ -74,7 +73,6 @@ class Reblogs extends ImmutablePureComponent {
 
         <ScrollableList
           scrollKey='reblogs'
-          shouldUpdateScroll={shouldUpdateScroll}
           emptyMessage={emptyMessage}
           bindToDocument={!multiColumn}
         >

app/javascript/mastodon/features/status/index.js

@@ -45,7 +45,7 @@ import { initBlockModal } from '../../actions/blocks';
 import { initBoostModal } from '../../actions/boosts';
 import { initReport } from '../../actions/reports';
 import { makeGetStatus, makeGetPictureInPicture } from '../../selectors';
-import { ScrollContainer } from 'react-router-scroll-4';
+import ScrollContainer from 'mastodon/containers/scroll_container';
 import ColumnBackButton from '../../components/column_back_button';
 import ColumnHeader from '../../components/column_header';
 import StatusContainer from '../../containers/status_container';
@@ -83,7 +83,7 @@ const makeMapStateToProps = () => {
     ancestorsIds = ancestorsIds.withMutations(mutable => {
       let id = statusId;
 
-      while (id) {
+      while (id && !mutable.includes(id)) {
         mutable.unshift(id);
         id = inReplyTos.get(id);
       }
@@ -101,7 +101,7 @@ const makeMapStateToProps = () => {
     const ids = [statusId];
 
     while (ids.length > 0) {
-      let id        = ids.shift();
+      let id        = ids.pop();
       const replies = contextReplies.get(id);
 
       if (statusId !== id) {
@@ -110,7 +110,7 @@ const makeMapStateToProps = () => {
 
       if (replies) {
         replies.reverse().forEach(reply => {
-          ids.unshift(reply);
+          if (!ids.includes(reply) && !descendantsIds.includes(reply) && statusId !== reply) ids.push(reply);
         });
       }
     }
@@ -498,7 +498,7 @@ class Status extends ImmutablePureComponent {
 
   render () {
     let ancestors, descendants;
-    const { shouldUpdateScroll, status, ancestorsIds, descendantsIds, intl, domain, multiColumn, pictureInPicture } = this.props;
+    const { status, ancestorsIds, descendantsIds, intl, domain, multiColumn, pictureInPicture } = this.props;
     const { fullscreen } = this.state;
 
     if (status === null) {
@@ -541,7 +541,7 @@ class Status extends ImmutablePureComponent {
           )}
         />
 
-        <ScrollContainer scrollKey='thread' shouldUpdateScroll={shouldUpdateScroll}>
+        <ScrollContainer scrollKey='thread'>
           <div className={classNames('scrollable', { fullscreen })} ref={this.setRef}>
             {ancestors}
 

app/javascript/mastodon/features/ui/components/audio_modal.js

@@ -4,7 +4,6 @@ import PropTypes from 'prop-types';
 import Audio from 'mastodon/features/audio';
 import { connect } from 'react-redux';
 import ImmutablePureComponent from 'react-immutable-pure-component';
-import { previewState } from './video_modal';
 import Footer from 'mastodon/features/picture_in_picture/components/footer';
 
 const mapStateToProps = (state, { statusId }) => ({
@@ -25,32 +24,6 @@ class AudioModal extends ImmutablePureComponent {
     onChangeBackgroundColor: PropTypes.func.isRequired,
   };
 
-  static contextTypes = {
-    router: PropTypes.object,
-  };
-
-  componentDidMount () {
-    if (this.context.router) {
-      const history = this.context.router.history;
-
-      history.push(history.location.pathname, previewState);
-
-      this.unlistenHistory = history.listen(() => {
-        this.props.onClose();
-      });
-    }
-  }
-
-  componentWillUnmount () {
-    if (this.context.router) {
-      this.unlistenHistory();
-
-      if (this.context.router.history.location.state === previewState) {
-        this.context.router.history.goBack();
-      }
-    }
-  }
-
   render () {
     const { media, accountStaticAvatar, statusId, onClose } = this.props;
     const options = this.props.options || {};

app/javascript/mastodon/features/ui/components/confirmation_modal.js

@@ -13,15 +13,22 @@ class ConfirmationModal extends React.PureComponent {
     onConfirm: PropTypes.func.isRequired,
     secondary: PropTypes.string,
     onSecondary: PropTypes.func,
+    closeWhenConfirm: PropTypes.bool,
     intl: PropTypes.object.isRequired,
   };
 
+  static defaultProps = {
+    closeWhenConfirm: true,
+  };
+
   componentDidMount() {
     this.button.focus();
   }
 
   handleClick = () => {
-    this.props.onClose();
+    if (this.props.closeWhenConfirm) {
+      this.props.onClose();
+    }
     this.props.onConfirm();
   }
 

app/javascript/mastodon/features/ui/components/link_footer.js

@@ -17,6 +17,7 @@ const mapDispatchToProps = (dispatch, { intl }) => ({
     dispatch(openModal('CONFIRM', {
       message: intl.formatMessage(messages.logoutMessage),
       confirm: intl.formatMessage(messages.logoutConfirm),
+      closeWhenConfirm: false,
       onConfirm: () => logOut(),
     }));
   },

app/javascript/mastodon/features/ui/components/media_modal.js

@@ -20,8 +20,6 @@ const messages = defineMessages({
   next: { id: 'lightbox.next', defaultMessage: 'Next' },
 });
 
-export const previewState = 'previewMediaModal';
-
 export default @injectIntl
 class MediaModal extends ImmutablePureComponent {
 
@@ -37,10 +35,6 @@ class MediaModal extends ImmutablePureComponent {
     volume: PropTypes.number,
   };
 
-  static contextTypes = {
-    router: PropTypes.object,
-  };
-
   state = {
     index: null,
     navigationHidden: false,
@@ -98,16 +92,6 @@ class MediaModal extends ImmutablePureComponent {
   componentDidMount () {
     window.addEventListener('keydown', this.handleKeyDown, false);
 
-    if (this.context.router) {
-      const history = this.context.router.history;
-
-      history.push(history.location.pathname, previewState);
-
-      this.unlistenHistory = history.listen(() => {
-        this.props.onClose();
-      });
-    }
-
     this._sendBackgroundColor();
   }
 
@@ -131,14 +115,6 @@ class MediaModal extends ImmutablePureComponent {
   componentWillUnmount () {
     window.removeEventListener('keydown', this.handleKeyDown);
 
-    if (this.context.router) {
-      this.unlistenHistory();
-
-      if (this.context.router.history.location.state === previewState) {
-        this.context.router.history.goBack();
-      }
-    }
-
     this.props.onChangeBackgroundColor(null);
   }
 

app/javascript/mastodon/features/ui/components/video_modal.js

@@ -6,8 +6,6 @@ import ImmutablePureComponent from 'react-immutable-pure-component';
 import Footer from 'mastodon/features/picture_in_picture/components/footer';
 import { getAverageFromBlurhash } from 'mastodon/blurhash';
 
-export const previewState = 'previewVideoModal';
-
 export default class VideoModal extends ImmutablePureComponent {
 
   static propTypes = {
@@ -22,19 +20,9 @@ export default class VideoModal extends ImmutablePureComponent {
     onChangeBackgroundColor: PropTypes.func.isRequired,
   };
 
-  static contextTypes = {
-    router: PropTypes.object,
-  };
-
   componentDidMount () {
-    const { router } = this.context;
     const { media, onChangeBackgroundColor, onClose } = this.props;
 
-    if (router) {
-      router.history.push(router.history.location.pathname, previewState);
-      this.unlistenHistory = router.history.listen(() => onClose());
-    }
-
     const backgroundColor = getAverageFromBlurhash(media.get('blurhash'));
 
     if (backgroundColor) {
@@ -42,18 +30,6 @@ export default class VideoModal extends ImmutablePureComponent {
     }
   }
 
-  componentWillUnmount () {
-    const { router } = this.context;
-
-    if (router) {
-      this.unlistenHistory();
-
-      if (router.history.location.state === previewState) {
-        router.history.goBack();
-      }
-    }
-  }
-
   render () {
     const { media, statusId, onClose } = this.props;
     const options = this.props.options || {};

app/javascript/mastodon/features/ui/containers/modal_container.js

@@ -3,8 +3,8 @@ import { closeModal } from '../../../actions/modal';
 import ModalRoot from '../components/modal_root';
 
 const mapStateToProps = state => ({
-  type: state.get('modal').modalType,
-  props: state.get('modal').modalProps,
+  type: state.getIn(['modal', 0, 'modalType'], null),
+  props: state.getIn(['modal', 0, 'modalProps'], {}),
 });
 
 const mapDispatchToProps = dispatch => ({

app/javascript/mastodon/features/ui/index.js

@@ -54,8 +54,6 @@ import {
   FollowRecommendations,
 } from './util/async-components';
 import { me } from '../../initial_state';
-import { previewState as previewMediaState } from './components/media_modal';
-import { previewState as previewVideoState } from './components/video_modal';
 import { closeOnboarding, INTRODUCTION_VERSION } from 'mastodon/actions/onboarding';
 
 // Dummy import, to make sure that <Status /> ends up in the application bundle.
@@ -138,10 +136,6 @@ class SwitchingColumnsArea extends React.PureComponent {
     }
   }
 
-  shouldUpdateScroll (_, { location }) {
-    return location.state !== previewMediaState && location.state !== previewVideoState;
-  }
-
   setRef = c => {
     if (c) {
       this.node = c.getWrappedInstance();
@@ -158,38 +152,38 @@ class SwitchingColumnsArea extends React.PureComponent {
           {redirect}
           <WrappedRoute path='/getting-started' component={GettingStarted} content={children} />
           <WrappedRoute path='/keyboard-shortcuts' component={KeyboardShortcuts} content={children} />
-          <WrappedRoute path='/timelines/home' component={HomeTimeline} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
-          <WrappedRoute path='/timelines/public' exact component={PublicTimeline} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
-          <WrappedRoute path='/timelines/public/local' exact component={CommunityTimeline} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
-          <WrappedRoute path='/timelines/direct' component={DirectTimeline} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
-          <WrappedRoute path='/timelines/tag/:id' component={HashtagTimeline} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
-          <WrappedRoute path='/timelines/list/:id' component={ListTimeline} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
+          <WrappedRoute path='/timelines/home' component={HomeTimeline} content={children} />
+          <WrappedRoute path='/timelines/public' exact component={PublicTimeline} content={children} />
+          <WrappedRoute path='/timelines/public/local' exact component={CommunityTimeline} content={children} />
+          <WrappedRoute path='/timelines/direct' component={DirectTimeline} content={children} />
+          <WrappedRoute path='/timelines/tag/:id' component={HashtagTimeline} content={children} />
+          <WrappedRoute path='/timelines/list/:id' component={ListTimeline} content={children} />
 
-          <WrappedRoute path='/notifications' component={Notifications} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
-          <WrappedRoute path='/favourites' component={FavouritedStatuses} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
+          <WrappedRoute path='/notifications' component={Notifications} content={children} />
+          <WrappedRoute path='/favourites' component={FavouritedStatuses} content={children} />
           <WrappedRoute path='/bookmarks' component={BookmarkedStatuses} content={children} />
-          <WrappedRoute path='/pinned' component={PinnedStatuses} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
+          <WrappedRoute path='/pinned' component={PinnedStatuses} content={children} />
 
           <WrappedRoute path='/start' component={FollowRecommendations} content={children} />
           <WrappedRoute path='/search' component={Search} content={children} />
-          <WrappedRoute path='/directory' component={Directory} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
+          <WrappedRoute path='/directory' component={Directory} content={children} />
 
           <WrappedRoute path='/statuses/new' component={Compose} content={children} />
-          <WrappedRoute path='/statuses/:statusId' exact component={Status} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
-          <WrappedRoute path='/statuses/:statusId/reblogs' component={Reblogs} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
-          <WrappedRoute path='/statuses/:statusId/favourites' component={Favourites} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
+          <WrappedRoute path='/statuses/:statusId' exact component={Status} content={children} />
+          <WrappedRoute path='/statuses/:statusId/reblogs' component={Reblogs} content={children} />
+          <WrappedRoute path='/statuses/:statusId/favourites' component={Favourites} content={children} />
 
-          <WrappedRoute path='/accounts/:accountId' exact component={AccountTimeline} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
-          <WrappedRoute path='/accounts/:accountId/with_replies' component={AccountTimeline} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll, withReplies: true }} />
-          <WrappedRoute path='/accounts/:accountId/followers' component={Followers} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
-          <WrappedRoute path='/accounts/:accountId/following' component={Following} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
-          <WrappedRoute path='/accounts/:accountId/media' component={AccountGallery} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
+          <WrappedRoute path='/accounts/:accountId' exact component={AccountTimeline} content={children} />
+          <WrappedRoute path='/accounts/:accountId/with_replies' component={AccountTimeline} content={children} componentParams={{ withReplies: true }} />
+          <WrappedRoute path='/accounts/:accountId/followers' component={Followers} content={children} />
+          <WrappedRoute path='/accounts/:accountId/following' component={Following} content={children} />
+          <WrappedRoute path='/accounts/:accountId/media' component={AccountGallery} content={children} />
 
-          <WrappedRoute path='/follow_requests' component={FollowRequests} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
-          <WrappedRoute path='/blocks' component={Blocks} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
-          <WrappedRoute path='/domain_blocks' component={DomainBlocks} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
-          <WrappedRoute path='/mutes' component={Mutes} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
-          <WrappedRoute path='/lists' component={Lists} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
+          <WrappedRoute path='/follow_requests' component={FollowRequests} content={children} />
+          <WrappedRoute path='/blocks' component={Blocks} content={children} />
+          <WrappedRoute path='/domain_blocks' component={DomainBlocks} content={children} />
+          <WrappedRoute path='/mutes' component={Mutes} content={children} />
+          <WrappedRoute path='/lists' component={Lists} content={children} />
 
           <WrappedRoute component={GenericNotFound} content={children} />
         </WrappedSwitch>

app/javascript/mastodon/reducers/modal.js

@@ -1,19 +1,15 @@
 import { MODAL_OPEN, MODAL_CLOSE } from '../actions/modal';
 import { TIMELINE_DELETE } from '../actions/timelines';
+import { Stack as ImmutableStack, Map as ImmutableMap } from 'immutable';
 
-const initialState = {
-  modalType: null,
-  modalProps: {},
-};
-
-export default function modal(state = initialState, action) {
+export default function modal(state = ImmutableStack(), action) {
   switch(action.type) {
   case MODAL_OPEN:
-    return { modalType: action.modalType, modalProps: action.modalProps };
+        return state.unshift(ImmutableMap({ modalType: action.modalType, modalProps: action.modalProps }));
   case MODAL_CLOSE:
-    return (action.modalType === undefined || action.modalType === state.modalType) ? initialState : state;
+    return (action.modalType === undefined || action.modalType === state.getIn([0, 'modalType'])) ? state.shift() : state;
   case TIMELINE_DELETE:
-    return (state.modalProps.statusId === action.id) ? initialState : state;
+        return state.filterNot((modal) => modal.get('modalProps').statusId === action.id);
   default:
     return state;
   }

app/javascript/mastodon/reducers/picture_in_picture.js

@@ -1,4 +1,5 @@
 import { PICTURE_IN_PICTURE_DEPLOY, PICTURE_IN_PICTURE_REMOVE } from 'mastodon/actions/picture_in_picture';
+import { TIMELINE_DELETE } from '../actions/timelines';
 
 const initialState = {
   statusId: null,
@@ -16,6 +17,8 @@ export default function pictureInPicture(state = initialState, action) {
     return { statusId: action.statusId, accountId: action.accountId, type: action.playerType, ...action.props };
   case PICTURE_IN_PICTURE_REMOVE:
     return { ...initialState };
+  case TIMELINE_DELETE:
+    return (state.statusId === action.id) ? { ...initialState } : state;
   default:
     return state;
   }

app/javascript/styles/mastodon/admin.scss

@@ -829,6 +829,7 @@ a.name-tag,
     padding: 0 5px;
     margin-bottom: 10px;
     flex: 1 0 50%;
+    max-width: 100%;
   }
 
   .account__header__fields,

app/javascript/styles/mastodon/components.scss

@@ -7284,6 +7284,7 @@ noscript {
     &__account {
       display: flex;
       text-decoration: none;
+      overflow: hidden;
     }
 
     .account__avatar {

app/lib/activitypub/activity/add.rb

@@ -7,7 +7,7 @@ class ActivityPub::Activity::Add < ActivityPub::Activity
     status   = status_from_uri(object_uri)
     status ||= fetch_remote_original_status
 
-    return unless !status.nil? && status.account_id == @account.id && !@account.pinned?(status)
+    return unless !status.nil? && status.account_id == @account.id && !@account.pinned?(status) && status.distributable?
 
     StatusPin.create!(account: @account, status: status)
   end

app/lib/activitypub/activity/create.rb

@@ -446,10 +446,14 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def supported_blurhash?(blurhash)
-    components = blurhash.blank? ? nil : Blurhash.components(blurhash)
+    components = blurhash.blank? || !blurhash_valid_chars?(blurhash) ? nil : Blurhash.components(blurhash)
     components.present? && components.none? { |comp| comp > 5 }
   end
 
+  def blurhash_valid_chars?(blurhash)
+    /^[\w#$%*+-.:;=?@\[\]^{|}~]+$/.match?(blurhash)
+  end
+
   def skip_download?
     return @skip_download if defined?(@skip_download)
 

app/lib/activitypub/adapter.rb

@@ -1,30 +1,7 @@
 # frozen_string_literal: true
 
 class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
-  NAMED_CONTEXT_MAP = {
-    activitystreams: 'https://www.w3.org/ns/activitystreams',
-    security: 'https://w3id.org/security/v1',
-  }.freeze
-
-  CONTEXT_EXTENSION_MAP = {
-    manually_approves_followers: { 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers' },
-    sensitive: { 'sensitive' => 'as:sensitive' },
-    hashtag: { 'Hashtag' => 'as:Hashtag' },
-    moved_to: { 'movedTo' => { '@id' => 'as:movedTo', '@type' => '@id' } },
-    also_known_as: { 'alsoKnownAs' => { '@id' => 'as:alsoKnownAs', '@type' => '@id' } },
-    emoji: { 'toot' => 'http://joinmastodon.org/ns#', 'Emoji' => 'toot:Emoji' },
-    featured: { 'toot' => 'http://joinmastodon.org/ns#', 'featured' => { '@id' => 'toot:featured', '@type' => '@id' }, 'featuredTags' => { '@id' => 'toot:featuredTags', '@type' => '@id' } },
-    property_value: { 'schema' => 'http://schema.org#', 'PropertyValue' => 'schema:PropertyValue', 'value' => 'schema:value' },
-    atom_uri: { 'ostatus' => 'http://ostatus.org#', 'atomUri' => 'ostatus:atomUri' },
-    conversation: { 'ostatus' => 'http://ostatus.org#', 'inReplyToAtomUri' => 'ostatus:inReplyToAtomUri', 'conversation' => 'ostatus:conversation' },
-    focal_point: { 'toot' => 'http://joinmastodon.org/ns#', 'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' } },
-    identity_proof: { 'toot' => 'http://joinmastodon.org/ns#', 'IdentityProof' => 'toot:IdentityProof' },
-    blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' },
-    discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' },
-    voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' },
-    olm: { 'toot' => 'http://joinmastodon.org/ns#', 'Device' => 'toot:Device', 'Ed25519Signature' => 'toot:Ed25519Signature', 'Ed25519Key' => 'toot:Ed25519Key', 'Curve25519Key' => 'toot:Curve25519Key', 'EncryptedMessage' => 'toot:EncryptedMessage', 'publicKeyBase64' => 'toot:publicKeyBase64', 'deviceId' => 'toot:deviceId', 'claim' => { '@type' => '@id', '@id' => 'toot:claim' }, 'fingerprintKey' => { '@type' => '@id', '@id' => 'toot:fingerprintKey' }, 'identityKey' => { '@type' => '@id', '@id' => 'toot:identityKey' }, 'devices' => { '@type' => '@id', '@id' => 'toot:devices' }, 'messageFranking' => 'toot:messageFranking', 'messageType' => 'toot:messageType', 'cipherText' => 'toot:cipherText' },
-    suspended: { 'toot' => 'http://joinmastodon.org/ns#', 'suspended' => 'toot:suspended' },
-  }.freeze
+  include ContextHelper
 
   def self.default_key_transform
     :camel_lower
@@ -35,7 +12,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
   end
 
   def serializable_hash(options = nil)
-    named_contexts     = {}
+    named_contexts     = { activitystreams: NAMED_CONTEXT_MAP['activitystreams'] }
     context_extensions = {}
 
     options         = serialization_options(options)
@@ -45,29 +22,4 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
 
     { '@context' => serialized_context(named_contexts, context_extensions) }.merge(serialized_hash)
   end
-
-  private
-
-  def serialized_context(named_contexts_map, context_extensions_map)
-    context_array = []
-
-    named_contexts     = [:activitystreams] + named_contexts_map.keys
-    context_extensions = context_extensions_map.keys
-
-    named_contexts.each do |key|
-      context_array << NAMED_CONTEXT_MAP[key]
-    end
-
-    extensions = context_extensions.each_with_object({}) do |key, h|
-      h.merge!(CONTEXT_EXTENSION_MAP[key])
-    end
-
-    context_array << extensions unless extensions.empty?
-
-    if context_array.size == 1
-      context_array.first
-    else
-      context_array
-    end
-  end
 end

app/lib/activitypub/tag_manager.rb

@@ -64,6 +64,10 @@ class ActivityPub::TagManager
     account_status_replies_url(target.account, target, page_params)
   end
 
+  def followers_uri_for(target)
+    target.local? ? account_followers_url(target) : target.followers_url.presence
+  end
+
   # Primary audience of a status
   # Public statuses go out to primarily the public collection
   # Unlisted and private statuses go out primarily to the followers collection
@@ -80,17 +84,17 @@ class ActivityPub::TagManager
         account_ids = status.active_mentions.pluck(:account_id)
         to = status.account.followers.where(id: account_ids).each_with_object([]) do |account, result|
           result << uri_for(account)
-          result << account_followers_url(account) if account.group?
+          result << followers_uri_for(account) if account.group?
         end
         to.concat(FollowRequest.where(target_account_id: status.account_id, account_id: account_ids).each_with_object([]) do |request, result|
           result << uri_for(request.account)
-          result << account_followers_url(request.account) if request.account.group?
-        end)
+          result << followers_uri_for(request.account) if request.account.group?
+        end).compact
       else
         status.active_mentions.each_with_object([]) do |mention, result|
           result << uri_for(mention.account)
-          result << account_followers_url(mention.account) if mention.account.group?
-        end
+          result << followers_uri_for(mention.account) if mention.account.group?
+        end.compact
       end
     end
   end
@@ -118,17 +122,17 @@ class ActivityPub::TagManager
         account_ids = status.active_mentions.pluck(:account_id)
         cc.concat(status.account.followers.where(id: account_ids).each_with_object([]) do |account, result|
           result << uri_for(account)
-          result << account_followers_url(account) if account.group?
-        end)
+          result << followers_uri_for(account) if account.group?
+        end.compact)
         cc.concat(FollowRequest.where(target_account_id: status.account_id, account_id: account_ids).each_with_object([]) do |request, result|
           result << uri_for(request.account)
-          result << account_followers_url(request.account) if request.account.group?
-        end)
+          result << followers_uri_for(request.account) if request.account.group?
+        end.compact)
       else
         cc.concat(status.active_mentions.each_with_object([]) do |mention, result|
           result << uri_for(mention.account)
-          result << account_followers_url(mention.account) if mention.account.group?
-        end)
+          result << followers_uri_for(mention.account) if mention.account.group?
+        end.compact)
       end
     end
 

app/lib/application_extension.rb

@@ -8,4 +8,8 @@ module ApplicationExtension
     validates :website, url: true, length: { maximum: 2_000 }, if: :website?
     validates :redirect_uri, length: { maximum: 2_000 }
   end
+
+  def confirmation_redirect_uri
+    redirect_uri.lines.first.strip
+  end
 end

app/lib/extractor.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module Extractor
+  MAX_DOMAIN_LENGTH = 253
+
   extend Twitter::TwitterText::Extractor
 
   module_function
@@ -14,7 +16,12 @@ module Extractor
     text.to_s.scan(Account::MENTION_RE) do |screen_name, _|
       match_data = $LAST_MATCH_INFO
       after = $'
+
       unless Twitter::TwitterText::Regex[:end_mention_match].match?(after)
+        username, domain = screen_name.split('@')
+
+        next if domain.present? && domain.length > MAX_DOMAIN_LENGTH
+
         start_position = match_data.char_begin(1) - 1
         end_position = match_data.char_end(1)
         possible_entries << {

app/lib/formatter.rb

@@ -8,6 +8,8 @@ class Formatter
 
   include ActionView::Helpers::TextHelper
 
+  DISALLOWED_BOUNDING_REGEX = /[[:alnum:]:]/.freeze
+
   def format(status, **options)
     if status.reblog?
       prepend_reblog = status.reblog.account.acct
@@ -140,54 +142,49 @@ class Formatter
 
     emoji_map = emojis.each_with_object({}) { |e, h| h[e.shortcode] = [full_asset_url(e.image.url), full_asset_url(e.image.url(:static))] }
 
-    i                     = -1
-    tag_open_index        = nil
-    inside_shortname      = false
-    shortname_start_index = -1
-    invisible_depth       = 0
+    tree = Nokogiri::HTML.fragment(html)
+    tree.xpath('./text()|.//text()[not(ancestor[@class="invisible"])]').to_a.each do |node|
+      i                     = -1
+      inside_shortname      = false
+      shortname_start_index = -1
+      last_index            = 0
+      text                  = node.content
+      result                = Nokogiri::XML::NodeSet.new(tree.document)
 
-    while i + 1 < html.size
-      i += 1
+      while i + 1 < text.size
+        i += 1
 
-      if invisible_depth.zero? && inside_shortname && html[i] == ':'
-        shortcode = html[shortname_start_index + 1..i - 1]
-        emoji     = emoji_map[shortcode]
+        if inside_shortname && text[i] == ':'
+          inside_shortname = false
+          shortcode = text[shortname_start_index + 1..i - 1]
+          char_after = text[i + 1]
+
+          next unless (char_after.nil? || !DISALLOWED_BOUNDING_REGEX.match?(char_after)) && (emoji = emoji_map[shortcode])
 
-        if emoji
           original_url, static_url = emoji
-          replacement = begin
+
+          result << Nokogiri::XML::Text.new(text[last_index..shortname_start_index - 1], tree.document) if shortname_start_index.positive?
+
+          result << Nokogiri::HTML.fragment(
             if animate
               image_tag(original_url, draggable: false, class: 'emojione', alt: ":#{shortcode}:", title: ":#{shortcode}:")
             else
               image_tag(original_url, draggable: false, class: 'emojione custom-emoji', alt: ":#{shortcode}:", title: ":#{shortcode}:", data: { original: original_url, static: static_url })
             end
-          end
-          before_html = shortname_start_index.positive? ? html[0..shortname_start_index - 1] : ''
-          html        = before_html + replacement + html[i + 1..-1]
-          i          += replacement.size - (shortcode.size + 2) - 1
-        else
-          i -= 1
-        end
+          )
 
-        inside_shortname = false
-      elsif tag_open_index && html[i] == '>'
-        tag = html[tag_open_index..i]
-        tag_open_index = nil
-        if invisible_depth.positive?
-          invisible_depth += count_tag_nesting(tag)
-        elsif tag == '<span class="invisible">'
-          invisible_depth = 1
+          last_index = i + 1
+        elsif text[i] == ':' && (i.zero? || !DISALLOWED_BOUNDING_REGEX.match?(text[i - 1]))
+          inside_shortname = true
+          shortname_start_index = i
         end
-      elsif html[i] == '<'
-        tag_open_index   = i
-        inside_shortname = false
-      elsif !tag_open_index && html[i] == ':'
-        inside_shortname      = true
-        shortname_start_index = i
       end
+
+      result << Nokogiri::XML::Text.new(text[last_index..-1], tree.document)
+      node.replace(result)
     end
 
-    html
+    tree.to_html
   end
   # rubocop:enable Metrics/BlockNesting
 
@@ -214,39 +211,10 @@ class Formatter
     result.flatten.join
   end
 
-  UNICODE_ESCAPE_BLACKLIST_RE = /\p{Z}|\p{P}/
-
   def utf8_friendly_extractor(text, options = {})
-    old_to_new_index = [0]
-
-    escaped = text.chars.map do |c|
-      output = begin
-        if c.ord.to_s(16).length > 2 && !UNICODE_ESCAPE_BLACKLIST_RE.match?(c)
-          CGI.escape(c)
-        else
-          c
-        end
-      end
-
-      old_to_new_index << old_to_new_index.last + output.length
-
-      output
-    end.join
-
     # Note: I couldn't obtain list_slug with @user/list-name format
     # for mention so this requires additional check
-    special = Extractor.extract_urls_with_indices(escaped, options).map do |extract|
-      new_indices = [
-        old_to_new_index.find_index(extract[:indices].first),
-        old_to_new_index.find_index(extract[:indices].last),
-      ]
-
-      next extract.merge(
-        indices: new_indices,
-        url: text[new_indices.first..new_indices.last - 1]
-      )
-    end
-
+    special = Extractor.extract_urls_with_indices(text, options)
     standard = Extractor.extract_entities_with_indices(text, options)
     extra = Extractor.extract_extra_uris_with_indices(text, options)
 

app/lib/webfinger.rb

@@ -46,7 +46,9 @@ class Webfinger
   def body_from_webfinger(url = standard_url, use_fallback = true)
     webfinger_request(url).perform do |res|
       if res.code == 200
-        res.body_with_limit
+        body = res.body_with_limit
+        raise Webfinger::Error, "Request for #{@uri} returned empty response" if body.empty?
+        body
       elsif res.code == 404 && use_fallback
         body_from_host_meta
       elsif res.code == 410

app/models/account.rb

@@ -58,8 +58,9 @@ class Account < ApplicationRecord
     hub_url
   )
 
-  USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
-  MENTION_RE  = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[a-z0-9]+)?)/i
+  USERNAME_RE   = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
+  MENTION_RE    = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[[:word:]]+)?)/i
+  URL_PREFIX_RE = /\Ahttp(s?):\/\/[^\/]+/
 
   include AccountAssociations
   include AccountAvatar
@@ -232,11 +233,11 @@ class Account < ApplicationRecord
     suspended? && deletion_request.present?
   end
 
-  def suspend!(date: Time.now.utc, origin: :local)
+  def suspend!(date: Time.now.utc, origin: :local, block_email: true)
     transaction do
       create_deletion_request!
       update!(suspended_at: date, suspension_origin: origin)
-      create_canonical_email_block!
+      create_canonical_email_block! if block_email
     end
   end
 
@@ -295,7 +296,11 @@ class Account < ApplicationRecord
   end
 
   def fields
-    (self[:fields] || []).map { |f| Field.new(self, f) }
+    (self[:fields] || []).map do |f|
+      Field.new(self, f)
+    rescue
+      nil
+    end.compact
   end
 
   def fields_attributes=(attributes)
@@ -375,7 +380,7 @@ class Account < ApplicationRecord
   def synchronization_uri_prefix
     return 'local' if local?
 
-    @synchronization_uri_prefix ||= uri[/http(s?):\/\/[^\/]+\//]
+    @synchronization_uri_prefix ||= "#{uri[URL_PREFIX_RE]}/"
   end
 
   class Field < ActiveModelSerializers::Model
@@ -421,6 +426,9 @@ class Account < ApplicationRecord
   end
 
   class << self
+    DISALLOWED_TSQUERY_CHARACTERS = /['?\\:‘’]/.freeze
+    TEXTSEARCH = "(setweight(to_tsvector('simple', accounts.display_name), 'A') || setweight(to_tsvector('simple', accounts.username), 'B') || setweight(to_tsvector('simple', coalesce(accounts.domain, '')), 'C'))"
+
     def readonly_attributes
       super - %w(statuses_count following_count followers_count)
     end
@@ -431,70 +439,29 @@ class Account < ApplicationRecord
     end
 
     def search_for(terms, limit = 10, offset = 0)
-      textsearch, query = generate_query_for_search(terms)
+      tsquery = generate_query_for_search(terms)
 
       sql = <<-SQL.squish
         SELECT
           accounts.*,
-          ts_rank_cd(#{textsearch}, #{query}, 32) AS rank
+          ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank
         FROM accounts
-        WHERE #{query} @@ #{textsearch}
+        WHERE to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH}
           AND accounts.suspended_at IS NULL
           AND accounts.moved_to_account_id IS NULL
         ORDER BY rank DESC
-        LIMIT ? OFFSET ?
+        LIMIT :limit OFFSET :offset
       SQL
 
-      records = find_by_sql([sql, limit, offset])
+      records = find_by_sql([sql, limit: limit, offset: offset, tsquery: tsquery])
       ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
       records
     end
 
     def advanced_search_for(terms, account, limit = 10, following = false, offset = 0)
-      textsearch, query = generate_query_for_search(terms)
-
-      if following
-        sql = <<-SQL.squish
-          WITH first_degree AS (
-            SELECT target_account_id
-            FROM follows
-            WHERE account_id = ?
-            UNION ALL
-            SELECT ?
-          )
-          SELECT
-            accounts.*,
-            (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank
-          FROM accounts
-          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?)
-          WHERE accounts.id IN (SELECT * FROM first_degree)
-            AND #{query} @@ #{textsearch}
-            AND accounts.suspended_at IS NULL
-            AND accounts.moved_to_account_id IS NULL
-          GROUP BY accounts.id
-          ORDER BY rank DESC
-          LIMIT ? OFFSET ?
-        SQL
-
-        records = find_by_sql([sql, account.id, account.id, account.id, limit, offset])
-      else
-        sql = <<-SQL.squish
-          SELECT
-            accounts.*,
-            (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank
-          FROM accounts
-          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?) OR (accounts.id = f.target_account_id AND f.account_id = ?)
-          WHERE #{query} @@ #{textsearch}
-            AND accounts.suspended_at IS NULL
-            AND accounts.moved_to_account_id IS NULL
-          GROUP BY accounts.id
-          ORDER BY rank DESC
-          LIMIT ? OFFSET ?
-        SQL
-
-        records = find_by_sql([sql, account.id, account.id, limit, offset])
-      end
-
+      tsquery = generate_query_for_search(terms)
+      sql = advanced_search_for_sql_template(following)
+      records = find_by_sql([sql, id: account.id, limit: limit, offset: offset, tsquery: tsquery])
       ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
       records
     end
@@ -516,12 +483,55 @@ class Account < ApplicationRecord
 
     private
 
-    def generate_query_for_search(terms)
-      terms      = Arel.sql(connection.quote(terms.gsub(/['?\\:]/, ' ')))
-      textsearch = "(setweight(to_tsvector('simple', accounts.display_name), 'A') || setweight(to_tsvector('simple', accounts.username), 'B') || setweight(to_tsvector('simple', coalesce(accounts.domain, '')), 'C'))"
-      query      = "to_tsquery('simple', ''' ' || #{terms} || ' ''' || ':*')"
+    def generate_query_for_search(unsanitized_terms)
+      terms = unsanitized_terms.gsub(DISALLOWED_TSQUERY_CHARACTERS, ' ')
 
-      [textsearch, query]
+      # The final ":*" is for prefix search.
+      # The trailing space does not seem to fit any purpose, but `to_tsquery`
+      # behaves differently with and without a leading space if the terms start
+      # with `./`, `../`, or `.. `. I don't understand why, so, in doubt, keep
+      # the same query.
+      "' #{terms} ':*"
+    end
+
+    def advanced_search_for_sql_template(following)
+      if following
+        <<-SQL.squish
+          WITH first_degree AS (
+            SELECT target_account_id
+            FROM follows
+            WHERE account_id = :id
+            UNION ALL
+            SELECT :id
+          )
+          SELECT
+            accounts.*,
+            (count(f.id) + 1) * ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank
+          FROM accounts
+          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = :id)
+          WHERE accounts.id IN (SELECT * FROM first_degree)
+            AND to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH}
+            AND accounts.suspended_at IS NULL
+            AND accounts.moved_to_account_id IS NULL
+          GROUP BY accounts.id
+          ORDER BY rank DESC
+          LIMIT :limit OFFSET :offset
+        SQL
+      else
+        <<-SQL.squish
+          SELECT
+            accounts.*,
+            (count(f.id) + 1) * ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank
+          FROM accounts
+          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = :id) OR (accounts.id = f.target_account_id AND f.account_id = :id)
+          WHERE to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH}
+            AND accounts.suspended_at IS NULL
+            AND accounts.moved_to_account_id IS NULL
+          GROUP BY accounts.id
+          ORDER BY rank DESC
+          LIMIT :limit OFFSET :offset
+        SQL
+      end
     end
   end
 
@@ -570,7 +580,11 @@ class Account < ApplicationRecord
   def create_canonical_email_block!
     return unless local? && user_email.present?
 
-    CanonicalEmailBlock.create(reference_account: self, email: user_email)
+    begin
+      CanonicalEmailBlock.create(reference_account: self, email: user_email)
+    rescue ActiveRecord::RecordNotUnique
+      # A canonical e-mail block may already exist for the same e-mail
+    end
   end
 
   def destroy_canonical_email_block!

app/models/account_note.rb

@@ -17,4 +17,5 @@ class AccountNote < ApplicationRecord
   belongs_to :target_account, class_name: 'Account'
 
   validates :account_id, uniqueness: { scope: :target_account_id }
+  validates :comment, length: { maximum: 2_000 }
 end

app/models/canonical_email_block.rb

@@ -15,7 +15,7 @@ class CanonicalEmailBlock < ApplicationRecord
 
   belongs_to :reference_account, class_name: 'Account'
 
-  validates :canonical_email_hash, presence: true
+  validates :canonical_email_hash, presence: true, uniqueness: true
 
   def email=(email)
     self.canonical_email_hash = email_to_canonical_email_hash(email)

app/models/concerns/account_interactions.rb

@@ -251,10 +251,13 @@ module AccountInteractions
          .where('users.current_sign_in_at > ?', User::ACTIVE_DURATION.ago)
   end
 
-  def remote_followers_hash(url_prefix)
-    Rails.cache.fetch("followers_hash:#{id}:#{url_prefix}") do
+  def remote_followers_hash(url)
+    url_prefix = url[Account::URL_PREFIX_RE]
+    return if url_prefix.blank?
+
+    Rails.cache.fetch("followers_hash:#{id}:#{url_prefix}/") do
       digest = "\x00" * 32
-      followers.where(Account.arel_table[:uri].matches(url_prefix + '%', false, true)).pluck_each(:uri) do |uri|
+      followers.where(Account.arel_table[:uri].matches("#{Account.sanitize_sql_like(url_prefix)}/%", false, true)).or(followers.where(uri: url_prefix)).pluck_each(:uri) do |uri|
         Xorcist.xor!(digest, Digest::SHA256.digest(uri))
       end
       digest.unpack('H*')[0]

app/models/status.rb

@@ -96,15 +96,12 @@ class Status < ApplicationRecord
   scope :not_excluded_by_account, ->(account) { where.not(account_id: account.excluded_from_timeline_account_ids) }
   scope :not_domain_blocked_by_account, ->(account) { account.excluded_from_timeline_domains.blank? ? left_outer_joins(:account) : left_outer_joins(:account).where('accounts.domain IS NULL OR accounts.domain NOT IN (?)', account.excluded_from_timeline_domains) }
   scope :tagged_with_all, ->(tag_ids) {
-    Array(tag_ids).reduce(self) do |result, id|
+    Array(tag_ids).map(&:to_i).reduce(self) do |result, id|
       result.joins("INNER JOIN statuses_tags t#{id} ON t#{id}.status_id = statuses.id AND t#{id}.tag_id = #{id}")
     end
   }
   scope :tagged_with_none, ->(tag_ids) {
-    Array(tag_ids).reduce(self) do |result, id|
-      result.joins("LEFT OUTER JOIN statuses_tags t#{id} ON t#{id}.status_id = statuses.id AND t#{id}.tag_id = #{id}")
-            .where("t#{id}.tag_id IS NULL")
-    end
+    where('NOT EXISTS (SELECT * FROM statuses_tags forbidden WHERE forbidden.status_id = statuses.id AND forbidden.tag_id IN (?))', tag_ids)
   }
 
   cache_associated :application,
@@ -338,7 +335,7 @@ class Status < ApplicationRecord
     def from_text(text)
       return [] if text.blank?
 
-      text.scan(FetchLinkCardService::URL_PATTERN).map(&:first).uniq.filter_map do |url|
+      text.scan(FetchLinkCardService::URL_PATTERN).map(&:second).uniq.filter_map do |url|
         status = begin
           if TagManager.instance.local_url?(url)
             ActivityPub::TagManager.instance.uri_to_resource(url, Status)
@@ -426,7 +423,7 @@ class Status < ApplicationRecord
   end
 
   def decrement_counter_caches
-    return if direct_visibility?
+    return if direct_visibility? || new_record?
 
     account&.decrement_count!(:statuses_count)
     reblog&.decrement_count!(:reblogs_count) if reblog?

app/models/user.rb

@@ -63,7 +63,7 @@ class User < ApplicationRecord
   devise :two_factor_backupable,
          otp_number_of_backup_codes: 10
 
-  devise :registerable, :recoverable, :rememberable, :validatable,
+  devise :registerable, :recoverable, :validatable,
          :confirmable
 
   include Omniauthable
@@ -86,11 +86,11 @@ class User < ApplicationRecord
   validates :invite_request, presence: true, on: :create, if: :invite_text_required?
 
   validates :locale, inclusion: I18n.available_locales.map(&:to_s), if: :locale?
-  validates_with BlacklistedEmailValidator, on: :create
+  validates_with BlacklistedEmailValidator, if: -> { !confirmed? }
   validates_with EmailMxValidator, if: :validate_email_dns?
   validates :agreement, acceptance: { allow_nil: false, accept: [true, 'true', '1'] }, on: :create
 
-  # Those are honeypot/antispam fields
+  # Honeypot/anti-spam fields
   attr_accessor :registration_form_time, :website, :confirm_password
 
   validates_with RegistrationFormTimeValidator, on: :create

app/serializers/manifest_serializer.rb

@@ -48,7 +48,7 @@ class ManifestSerializer < ActiveModel::Serializer
   end
 
   def scope
-    root_url
+    '/'
   end
 
   def share_target

app/serializers/rest/instance_serializer.rb

@@ -5,7 +5,8 @@ class REST::InstanceSerializer < ActiveModel::Serializer
 
   attributes :uri, :title, :short_description, :description, :email,
              :version, :urls, :stats, :thumbnail,
-             :languages, :registrations, :approval_required, :invites_enabled
+             :languages, :registrations, :approval_required, :invites_enabled,
+             :configuration
 
   has_one :contact_account, serializer: REST::AccountSerializer
 
@@ -53,6 +54,32 @@ class REST::InstanceSerializer < ActiveModel::Serializer
     { streaming_api: Rails.configuration.x.streaming_api_base_url }
   end
 
+  def configuration
+    {
+      statuses: {
+        max_characters: StatusLengthValidator::MAX_CHARS,
+        max_media_attachments: 4,
+        characters_reserved_per_url: StatusLengthValidator::URL_PLACEHOLDER_CHARS,
+      },
+
+      media_attachments: {
+        supported_mime_types: MediaAttachment::IMAGE_MIME_TYPES + MediaAttachment::VIDEO_MIME_TYPES + MediaAttachment::AUDIO_MIME_TYPES,
+        image_size_limit: MediaAttachment::IMAGE_LIMIT,
+        image_matrix_limit: Attachmentable::MAX_MATRIX_LIMIT,
+        video_size_limit: MediaAttachment::VIDEO_LIMIT,
+        video_frame_rate_limit: MediaAttachment::MAX_VIDEO_FRAME_RATE,
+        video_matrix_limit: MediaAttachment::MAX_VIDEO_MATRIX_LIMIT,
+      },
+
+      polls: {
+        max_options: PollValidator::MAX_OPTIONS,
+        max_characters_per_option: PollValidator::MAX_OPTION_CHARS,
+        min_expiration: PollValidator::MIN_EXPIRATION,
+        max_expiration: PollValidator::MAX_EXPIRATION,
+      },
+    }
+  end
+
   def languages
     [I18n.default_locale]
   end

app/services/activitypub/process_collection_service.rb

@@ -5,11 +5,27 @@ class ActivityPub::ProcessCollectionService < BaseService
 
   def call(body, account, **options)
     @account = account
-    @json    = Oj.load(body, mode: :strict)
+    @json    = original_json = Oj.load(body, mode: :strict)
     @options = options
 
+    begin
+      @json = compact(@json) if @json['signature'].is_a?(Hash)
+    rescue JSON::LD::JsonLdError => e
+      Rails.logger.debug "Error when compacting JSON-LD document for #{value_or_id(@json['actor'])}: #{e.message}"
+      @json = original_json.without('signature')
+    end
+
     return if !supported_context? || (different_actor? && verify_account!.nil?) || suspended_actor? || @account.local?
 
+    if @json['signature'].present?
+      # We have verified the signature, but in the compaction step above, might
+      # have introduced incompatibilities with other servers that do not
+      # normalize the JSON-LD documents (for instance, previous Mastodon
+      # versions), so skip redistribution if we can't get a safe document.
+      patch_for_forwarding!(original_json, @json)
+      @json.delete('signature') unless safe_for_forwarding?(original_json, @json)
+    end
+
     case @json['type']
     when 'Collection', 'CollectionPage'
       process_items @json['items']

app/services/fetch_oembed_service.rb

@@ -2,6 +2,7 @@
 
 class FetchOEmbedService
   ENDPOINT_CACHE_EXPIRES_IN = 24.hours.freeze
+  URL_REGEX                 = /(=(http[s]?(%3A|:)(\/\/|%2F%2F)))([^&]*)/i.freeze
 
   attr_reader :url, :options, :format, :endpoint_url
 
@@ -65,10 +66,12 @@ class FetchOEmbedService
   end
 
   def cache_endpoint!
+    return unless URL_REGEX.match?(@endpoint_url)
+
     url_domain = Addressable::URI.parse(@url).normalized_host
 
     endpoint_hash = {
-      endpoint: @endpoint_url.gsub(/(=(http[s]?(%3A|:)(\/\/|%2F%2F)))([^&]*)/i, '={url}'),
+      endpoint: @endpoint_url.gsub(URL_REGEX, '={url}'),
       format: @format,
     }
 

app/services/notify_service.rb

@@ -67,8 +67,53 @@ class NotifyService < BaseService
     message? && @notification.target_status.direct_visibility?
   end
 
+  # Returns true if the sender has been mentionned by the recipient up the thread
   def response_to_recipient?
-    @notification.target_status.in_reply_to_account_id == @recipient.id && @notification.target_status.thread&.direct_visibility?
+    return false if @notification.target_status.in_reply_to_id.nil?
+
+    # Using an SQL CTE to avoid unneeded back-and-forth with SQL server in case of long threads
+    !Status.count_by_sql([<<-SQL.squish, id: @notification.target_status.in_reply_to_id, recipient_id: @recipient.id, sender_id: @notification.from_account.id]).zero?
+      WITH RECURSIVE ancestors(id, in_reply_to_id, replying_to_sender, path) AS (
+          SELECT
+            s.id,
+            s.in_reply_to_id,
+            (CASE
+              WHEN s.account_id = :recipient_id THEN
+                EXISTS (
+                  SELECT *
+                  FROM mentions m
+                  WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
+                )
+              ELSE
+                FALSE
+             END),
+            ARRAY[s.id]
+          FROM statuses s
+          WHERE s.id = :id
+        UNION ALL
+          SELECT
+            s.id,
+            s.in_reply_to_id,
+            (CASE
+              WHEN s.account_id = :recipient_id THEN
+                EXISTS (
+                  SELECT *
+                  FROM mentions m
+                  WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
+                )
+              ELSE
+                FALSE
+             END),
+            st.path || s.id
+          FROM ancestors st
+          JOIN statuses s ON s.id = st.in_reply_to_id
+          WHERE st.replying_to_sender IS FALSE AND NOT s.id = ANY(path)
+      )
+      SELECT COUNT(*)
+      FROM ancestors st
+      JOIN statuses s ON s.id = st.id
+      WHERE st.replying_to_sender IS TRUE AND s.visibility = 3
+    SQL
   end
 
   def from_staff?

app/services/post_status_service.rb

@@ -74,6 +74,9 @@ class PostStatusService < BaseService
     status_for_validation = @account.statuses.build(status_attributes)
 
     if status_for_validation.valid?
+      # Marking the status as destroyed is necessary to prevent the status from being
+      # persisted when the associated media attachments get updated when creating the
+      # scheduled status.
       status_for_validation.destroy
 
       # The following transaction block is needed to wrap the UPDATEs to

app/services/resolve_account_service.rb

@@ -142,6 +142,7 @@ class ResolveAccountService < BaseService
   end
 
   def queue_deletion!
+    @account.suspend!(origin: :remote)
     AccountDeletionWorker.perform_async(@account.id, reserve_username: false, skip_activitypub: true)
   end