Bump version to v4.1.4

* Fix timeout handling of outbound HTTP requests

* Use CLOCK_MONOTONIC instead of Time.now

.github/workflows/build-image.yml

@@ -12,6 +12,7 @@ on:
       - Dockerfile
 permissions:
   contents: read
+  packages: write
 
 jobs:
   build-image:
@@ -26,15 +27,28 @@ jobs:
       - uses: hadolint/hadolint-action@v3.1.0
       - uses: docker/setup-qemu-action@v2
       - uses: docker/setup-buildx-action@v2
-      - uses: docker/login-action@v2
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-        if: github.event_name != 'pull_request'
+        if: github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request'
+
+      - name: Log in to the Github Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+        if: github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request'
+
       - uses: docker/metadata-action@v4
         id: meta
         with:
-          images: tootsuite/mastodon
+          images: |
+            tootsuite/mastodon
+            ghcr.io/mastodon/mastodon
           flavor: |
             latest=auto
           tags: |
@@ -42,13 +56,15 @@ jobs:
             type=pep440,pattern={{raw}}
             type=pep440,pattern=v{{major}}.{{minor}}
             type=ref,event=pr
+
       - uses: docker/build-push-action@v4
         with:
           context: .
           platforms: linux/amd64,linux/arm64
           provenance: false
           builder: ${{ steps.buildx.outputs.name }}
-          push: ${{ github.event_name != 'pull_request' }}
+          push: ${{ github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max

.ruby-version

@@ -1 +1 @@
-3.0.4
+3.0.6

CHANGELOG.md

@@ -3,6 +3,127 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
+## [4.1.4] - 2023-07-07
+
+### Fixed
+
+- Fix branding:generate_app_icons failing because of disallowed ICO coder ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25794))
+- Fix crash in admin interface when viewing a remote user with verified links ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25796))
+- Fix processing of media files with unusual names ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25788))
+
+## [4.1.3] - 2023-07-06
+
+### Added
+
+- Add fallback redirection when getting a webfinger query `LOCAL_DOMAIN@LOCAL_DOMAIN` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23600))
+
+### Changed
+
+- Change OpenGraph-based embeds to allow fullscreen ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25058))
+- Change AccessTokensVacuum to also delete expired tokens ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24868))
+- Change profile updates to be sent to recently-mentioned servers ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24852))
+- Change automatic post deletion thresholds and load detection ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24614))
+- Change `/api/v1/statuses/:id/history` to always return at least one item ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25510))
+- Change auto-linking to allow carets in URL query params ([renchap](https://github.com/mastodon/mastodon/pull/25216))
+
+### Removed
+
+- Remove invalid `X-Frame-Options: ALLOWALL` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25070))
+
+### Fixed
+
+- Fix wrong view being displayed when a webhook fails validation ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25464))
+- Fix soft-deleted post cleanup scheduler overwhelming the streaming server ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25519))
+- Fix incorrect pagination headers in `/api/v2/admin/accounts` ([danielmbrasil](https://github.com/mastodon/mastodon/pull/25477))
+- Fix multiple inefficiencies in automatic post cleanup worker ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24607), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24785), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24840))
+- Fix performance of streaming by parsing message JSON once ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25278), [ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25361))
+- Fix CSP headers when `S3_ALIAS_HOST` includes a path component ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25273))
+- Fix `tootctl accounts approve --number N` not approving N earliest registrations ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24605))
+- Fix reports not being closed when performing batch suspensions ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24988))
+- Fix being able to vote on your own polls ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25015))
+- Fix race condition when reblogging a status ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25016))
+- Fix “Authorized applications” inefficiently and incorrectly getting last use date ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25060))
+- Fix “Authorized applications” crashing when listing apps with certain admin API scopes ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25713))
+- Fix multiple N+1s in ConversationsController ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25134), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25399), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25499))
+- Fix user archive takeouts when using OpenStack Swift ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24431))
+- Fix searching for remote content by URL not working under certain conditions ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25637))
+- Fix inefficiencies in indexing content for search ([VyrCossont](https://github.com/mastodon/mastodon/pull/24285), [VyrCossont](https://github.com/mastodon/mastodon/pull/24342))
+
+### Security
+
+- Add finer permission requirements for managing webhooks ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25463))
+- Update dependencies
+- Add hardening headers for user-uploaded files ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25756))
+- Fix verified links possibly hiding important parts of the URL (CVE-2023-36462)
+- Fix timeout handling of outbound HTTP requests (CVE-2023-36461)
+- Fix arbitrary file creation through media processing (CVE-2023-36460)
+- Fix possible XSS in preview cards (CVE-2023-36459)
+
+## [4.1.2] - 2023-04-04
+
+### Fixed
+
+- Fix crash in `tootctl` commands making use of parallelization when Elasticsearch is enabled ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24182), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24377))
+- Fix crash in `db:setup` when Elasticsearch is enabled ([rrgeorge](https://github.com/mastodon/mastodon/pull/24302))
+- Fix user archive takeout when using OpenStack Swift or S3 providers with no ACL support ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24200))
+- Fix invalid/expired invites being processed on sign-up ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24337))
+
+### Security
+
+- Update Ruby to 3.0.6 due to ReDoS vulnerabilities ([saizai](https://github.com/mastodon/mastodon/pull/24334))
+- Fix unescaped user input in LDAP query ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24379))
+
+## [4.1.1] - 2023-03-16
+
+### Added
+
+- Add redirection from paths with url-encoded `@` to their decoded form ([thijskh](https://github.com/mastodon/mastodon/pull/23593))
+- Add `lang` attribute to native language names in language picker in Web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23749))
+- Add headers to outgoing mails to avoid auto-replies ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23597))
+- Add support for refreshing many accounts at once with `tootctl accounts refresh` ([9p4](https://github.com/mastodon/mastodon/pull/23304))
+- Add confirmation modal when clicking to edit a post with a non-empty compose form ([PauloVilarinho](https://github.com/mastodon/mastodon/pull/23936))
+- Add support for the HAproxy PROXY protocol through the `PROXY_PROTO_V1` environment variable ([CSDUMMI](https://github.com/mastodon/mastodon/pull/24064))
+- Add `SENDFILE_HEADER` environment variable ([Gargron](https://github.com/mastodon/mastodon/pull/24123))
+- Add cache headers to static files served through Rails ([Gargron](https://github.com/mastodon/mastodon/pull/24120))
+
+### Changed
+
+- Increase contrast of upload progress bar background ([toolmantim](https://github.com/mastodon/mastodon/pull/23836))
+- Change post auto-deletion throttling constants to better scale with server size ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23320))
+- Change order of bookmark and favourite sidebar entries in single-column UI for consistency ([TerryGarcia](https://github.com/mastodon/mastodon/pull/23701))
+- Change `ActivityPub::DeliveryWorker` retries to be spread out more ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/21956))
+
+### Fixed
+
+- Fix “Remove all followers from the selected domains” also removing follows and notifications ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23805))
+- Fix streaming metrics format ([emilweth](https://github.com/mastodon/mastodon/pull/23519), [emilweth](https://github.com/mastodon/mastodon/pull/23520))
+- Fix case-sensitive check for previously used hashtags in hashtag autocompletion ([deanveloper](https://github.com/mastodon/mastodon/pull/23526))
+- Fix focus point of already-attached media not saving after edit ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23566))
+- Fix sidebar behavior in settings/admin UI on mobile ([wxt2005](https://github.com/mastodon/mastodon/pull/23764))
+- Fix inefficiency when searching accounts per username in admin interface ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23801))
+- Fix duplicate “Publish” button on mobile ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23804))
+- Fix server error when failing to follow back followers from `/relationships` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23787))
+- Fix server error when attempting to display the edit history of a trendable post in the admin interface ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23574))
+- Fix `tootctl accounts migrate` crashing because of a typo ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23567))
+- Fix original account being unfollowed on migration before the follow request to the new account could be sent ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/21957))
+- Fix the “Back” button in column headers sometimes leaving Mastodon ([c960657](https://github.com/mastodon/mastodon/pull/23953))
+- Fix pgBouncer resetting application name on every transaction ([Gargron](https://github.com/mastodon/mastodon/pull/23958))
+- Fix unconfirmed accounts being counted as active users ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23803))
+- Fix `/api/v1/streaming` sub-paths not being redirected ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23988))
+- Fix drag'n'drop upload area text that spans multiple lines not being centered ([vintprox](https://github.com/mastodon/mastodon/pull/24029))
+- Fix sidekiq jobs not triggering Elasticsearch index updates ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24046))
+- Fix tags being unnecessarily stripped from plain-text short site description ([c960657](https://github.com/mastodon/mastodon/pull/23975))
+- Fix HTML entities not being un-escaped in extracted plain-text from remote posts ([c960657](https://github.com/mastodon/mastodon/pull/24019))
+- Fix dashboard crash on ElasticSearch server error ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23751))
+- Fix incorrect post links in strikes when the account is remote ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23611))
+- Fix misleading error code when receiving invalid WebAuthn credentials ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23568))
+- Fix duplicate mails being sent when the SMTP server is too slow to close the connection ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23750))
+
+### Security
+
+- Change user backups to use expiring URLs for download when possible ([Gargron](https://github.com/mastodon/mastodon/pull/24136))
+- Add warning for object storage misconfiguration ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24137))
+
 ## [4.1.0] - 2023-02-10
 
 ### Added

Dockerfile

@@ -2,7 +2,7 @@
 # This needs to be bullseye-slim because the Ruby image is built on bullseye-slim
 ARG NODE_VERSION="16.18.1-bullseye-slim"
 
-FROM ghcr.io/moritzheiber/ruby-jemalloc:3.0.4-slim as ruby
+FROM ghcr.io/moritzheiber/ruby-jemalloc:3.0.6-slim as ruby
 FROM node:${NODE_VERSION} as build
 
 COPY --link --from=ruby /opt/ruby /opt/ruby

Gemfile.lock

@@ -10,40 +10,40 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.1.7.2)
-      actionpack (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
+    actioncable (6.1.7.4)
+      actionpack (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.1.7.2)
-      actionpack (= 6.1.7.2)
-      activejob (= 6.1.7.2)
-      activerecord (= 6.1.7.2)
-      activestorage (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
+    actionmailbox (6.1.7.4)
+      actionpack (= 6.1.7.4)
+      activejob (= 6.1.7.4)
+      activerecord (= 6.1.7.4)
+      activestorage (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
       mail (>= 2.7.1)
-    actionmailer (6.1.7.2)
-      actionpack (= 6.1.7.2)
-      actionview (= 6.1.7.2)
-      activejob (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
+    actionmailer (6.1.7.4)
+      actionpack (= 6.1.7.4)
+      actionview (= 6.1.7.4)
+      activejob (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.1.7.2)
-      actionview (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
+    actionpack (6.1.7.4)
+      actionview (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
       rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.1.7.2)
-      actionpack (= 6.1.7.2)
-      activerecord (= 6.1.7.2)
-      activestorage (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
+    actiontext (6.1.7.4)
+      actionpack (= 6.1.7.4)
+      activerecord (= 6.1.7.4)
+      activestorage (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
       nokogiri (>= 1.8.5)
-    actionview (6.1.7.2)
-      activesupport (= 6.1.7.2)
+    actionview (6.1.7.4)
+      activesupport (= 6.1.7.4)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
@@ -54,22 +54,22 @@ GEM
       case_transform (>= 0.2)
       jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
     active_record_query_trace (1.8)
-    activejob (6.1.7.2)
-      activesupport (= 6.1.7.2)
+    activejob (6.1.7.4)
+      activesupport (= 6.1.7.4)
       globalid (>= 0.3.6)
-    activemodel (6.1.7.2)
-      activesupport (= 6.1.7.2)
-    activerecord (6.1.7.2)
-      activemodel (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
-    activestorage (6.1.7.2)
-      actionpack (= 6.1.7.2)
-      activejob (= 6.1.7.2)
-      activerecord (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
+    activemodel (6.1.7.4)
+      activesupport (= 6.1.7.4)
+    activerecord (6.1.7.4)
+      activemodel (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
+    activestorage (6.1.7.4)
+      actionpack (= 6.1.7.4)
+      activejob (= 6.1.7.4)
+      activerecord (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (6.1.7.2)
+    activesupport (6.1.7.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -120,8 +120,7 @@ GEM
     bindata (2.4.14)
     binding_of_caller (1.0.0)
       debug_inspector (>= 0.0.1)
-    blurhash (0.1.6)
-      ffi (~> 1.14)
+    blurhash (0.1.7)
     bootsnap (1.16.0)
       msgpack (~> 1.2)
     brakeman (5.4.0)
@@ -174,7 +173,7 @@ GEM
     cocoon (1.2.15)
     coderay (1.1.3)
     color_diff (0.1)
-    concurrent-ruby (1.2.0)
+    concurrent-ruby (1.2.2)
     connection_pool (2.3.0)
     cose (1.2.1)
       cbor (~> 0.5.9)
@@ -207,7 +206,7 @@ GEM
     docile (1.4.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    doorkeeper (5.6.4)
+    doorkeeper (5.6.6)
       railties (>= 5)
     dotenv (2.8.1)
     dotenv-rails (2.8.1)
@@ -389,7 +388,7 @@ GEM
     loofah (2.19.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
-    mail (2.8.0.1)
+    mail (2.8.1)
       mini_mime (>= 0.1.1)
       net-imap
       net-pop
@@ -406,12 +405,12 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2022.0105)
     mini_mime (1.1.2)
-    mini_portile2 (2.8.1)
+    mini_portile2 (2.8.2)
     minitest (5.17.0)
     msgpack (1.6.0)
     multi_json (1.15.0)
     multipart-post (2.1.1)
-    net-imap (0.3.4)
+    net-imap (0.3.6)
       date
       net-protocol
     net-ldap (0.17.1)
@@ -424,8 +423,8 @@ GEM
     net-smtp (0.3.3)
       net-protocol
     net-ssh (7.0.1)
-    nio4r (2.5.8)
-    nokogiri (1.14.1)
+    nio4r (2.5.9)
+    nokogiri (1.14.5)
       mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
     nsa (0.2.8)
@@ -498,7 +497,7 @@ GEM
       activesupport (>= 3.0.0)
     raabro (1.4.0)
     racc (1.6.2)
-    rack (2.2.6.2)
+    rack (2.2.7)
     rack-attack (6.6.1)
       rack (>= 1.0, < 3)
     rack-cors (1.1.1)
@@ -513,20 +512,20 @@ GEM
       rack
     rack-test (2.0.2)
       rack (>= 1.3)
-    rails (6.1.7.2)
-      actioncable (= 6.1.7.2)
-      actionmailbox (= 6.1.7.2)
-      actionmailer (= 6.1.7.2)
-      actionpack (= 6.1.7.2)
-      actiontext (= 6.1.7.2)
-      actionview (= 6.1.7.2)
-      activejob (= 6.1.7.2)
-      activemodel (= 6.1.7.2)
-      activerecord (= 6.1.7.2)
-      activestorage (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
+    rails (6.1.7.4)
+      actioncable (= 6.1.7.4)
+      actionmailbox (= 6.1.7.4)
+      actionmailer (= 6.1.7.4)
+      actionpack (= 6.1.7.4)
+      actiontext (= 6.1.7.4)
+      actionview (= 6.1.7.4)
+      activejob (= 6.1.7.4)
+      activemodel (= 6.1.7.4)
+      activerecord (= 6.1.7.4)
+      activestorage (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
       bundler (>= 1.15.0)
-      railties (= 6.1.7.2)
+      railties (= 6.1.7.4)
       sprockets-rails (>= 2.0.0)
     rails-controller-testing (1.0.5)
       actionpack (>= 5.0.1.rc1)
@@ -542,9 +541,9 @@ GEM
       railties (>= 6.0.0, < 7)
     rails-settings-cached (0.6.6)
       rails (>= 4.2.0)
-    railties (6.1.7.2)
-      actionpack (= 6.1.7.2)
-      activesupport (= 6.1.7.2)
+    railties (6.1.7.4)
+      actionpack (= 6.1.7.4)
+      activesupport (= 6.1.7.4)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
@@ -628,7 +627,7 @@ GEM
       fugit (~> 1.1, >= 1.1.6)
     safety_net_attestation (0.4.0)
       jwt (~> 2.0)
-    sanitize (6.0.1)
+    sanitize (6.0.2)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     scenic (1.7.0)
@@ -689,9 +688,9 @@ GEM
       unicode-display_width (>= 1.1.1, < 3)
     terrapin (0.6.0)
       climate_control (>= 0.0.3, < 1.0)
-    thor (1.2.1)
+    thor (1.2.2)
     tilt (2.0.11)
-    timeout (0.3.1)
+    timeout (0.3.2)
     tpm-key_attestation (0.11.0)
       bindata (~> 2.4)
       openssl (> 2.0, < 3.1)
@@ -754,7 +753,7 @@ GEM
     xorcist (1.1.3)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.6.6)
+    zeitwerk (2.6.8)
 
 PLATFORMS
   ruby

README.md

@@ -8,13 +8,11 @@
 [![Build Status](https://img.shields.io/circleci/project/github/mastodon/mastodon.svg)][circleci]
 [![Code Climate](https://img.shields.io/codeclimate/maintainability/mastodon/mastodon.svg)][code_climate]
 [![Crowdin](https://d322cqt584bo4o.cloudfront.net/mastodon/localized.svg)][crowdin]
-[![Docker Pulls](https://img.shields.io/docker/pulls/tootsuite/mastodon.svg)][docker]
 
 [releases]: https://github.com/mastodon/mastodon/releases
 [circleci]: https://circleci.com/gh/mastodon/mastodon
 [code_climate]: https://codeclimate.com/github/mastodon/mastodon
 [crowdin]: https://crowdin.com/project/mastodon
-[docker]: https://hub.docker.com/r/tootsuite/mastodon/
 
 Mastodon is a **free, open-source social network server** based on ActivityPub where users can follow friends and discover new ones. On Mastodon, users can publish anything they want: links, pictures, text, video. All Mastodon servers are interoperable as a federated network (users on one server can seamlessly communicate with users from another one, including non-Mastodon software that implements ActivityPub!)
 
@@ -31,6 +29,7 @@ Click below to **learn more** in a video:
 - [View sponsors](https://joinmastodon.org/sponsors)
 - [Blog](https://blog.joinmastodon.org)
 - [Documentation](https://docs.joinmastodon.org)
+- [Official Docker image](https://github.com/mastodon/mastodon/pkgs/container/mastodon)
 - [Browse Mastodon servers](https://joinmastodon.org/communities)
 - [Browse Mastodon apps](https://joinmastodon.org/apps)
 

app/controllers/admin/webhooks_controller.rb

@@ -20,6 +20,7 @@ module Admin
       authorize :webhook, :create?
 
       @webhook = Webhook.new(resource_params)
+      @webhook.current_account = current_account
 
       if @webhook.save
         redirect_to admin_webhook_path(@webhook)
@@ -39,10 +40,12 @@ module Admin
     def update
       authorize @webhook, :update?
 
+      @webhook.current_account = current_account
+
       if @webhook.update(resource_params)
         redirect_to admin_webhook_path(@webhook)
       else
-        render :show
+        render :edit
       end
     end
 

app/controllers/api/v1/conversations_controller.rb

@@ -11,7 +11,7 @@ class Api::V1::ConversationsController < Api::BaseController
 
   def index
     @conversations = paginated_conversations
-    render json: @conversations, each_serializer: REST::ConversationSerializer
+    render json: @conversations, each_serializer: REST::ConversationSerializer, relationships: StatusRelationshipsPresenter.new(@conversations.map(&:last_status), current_user&.account_id)
   end
 
   def read
@@ -32,6 +32,19 @@ class Api::V1::ConversationsController < Api::BaseController
 
   def paginated_conversations
     AccountConversation.where(account: current_account)
+                       .includes(
+                         account: :account_stat,
+                         last_status: [
+                           :media_attachments,
+                           :preview_cards,
+                           :status_stat,
+                           :tags,
+                           {
+                             active_mentions: [account: :account_stat],
+                             account: :account_stat,
+                           },
+                         ]
+                       )
                        .to_a_paginated_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id))
   end
 

app/controllers/api/v1/statuses/histories_controller.rb

@@ -7,11 +7,15 @@ class Api::V1::Statuses::HistoriesController < Api::BaseController
   before_action :set_status
 
   def show
-    render json: @status.edits.includes(:account, status: [:account]), each_serializer: REST::StatusEditSerializer
+    render json: status_edits, each_serializer: REST::StatusEditSerializer
   end
 
   private
 
+  def status_edits
+    @status.edits.includes(:account, status: [:account]).to_a.presence || [@status.build_snapshot(at_time: @status.edited_at || @status.created_at)]
+  end
+
   def set_status
     @status = Status.find(params[:status_id])
     authorize @status, :show?

app/controllers/api/v1/statuses/reblogs_controller.rb

@@ -2,6 +2,8 @@
 
 class Api::V1::Statuses::ReblogsController < Api::BaseController
   include Authorization
+  include Redisable
+  include Lockable
 
   before_action -> { doorkeeper_authorize! :write, :'write:statuses' }
   before_action :require_user!
@@ -10,7 +12,9 @@ class Api::V1::Statuses::ReblogsController < Api::BaseController
   override_rate_limit_headers :create, family: :statuses
 
   def create
-    @status = ReblogService.new.call(current_account, @reblog, reblog_params)
+    with_lock("reblog:#{current_account.id}:#{@reblog.id}") do
+      @status = ReblogService.new.call(current_account, @reblog, reblog_params)
+    end
 
     render json: @status, serializer: REST::StatusSerializer
   end

app/controllers/api/v2/admin/accounts_controller.rb

@@ -18,6 +18,14 @@ class Api::V2::Admin::AccountsController < Api::V1::Admin::AccountsController
 
   private
 
+  def next_path
+    api_v2_admin_accounts_url(pagination_params(max_id: pagination_max_id)) if records_continue?
+  end
+
+  def prev_path
+    api_v2_admin_accounts_url(pagination_params(min_id: pagination_since_id)) unless @accounts.empty?
+  end
+
   def filtered_accounts
     AccountFilter.new(translated_filter_params).results
   end

app/controllers/auth/registrations_controller.rb

@@ -48,7 +48,7 @@ class Auth::RegistrationsController < Devise::RegistrationsController
     super(hash)
 
     resource.locale                 = I18n.locale
-    resource.invite_code            = params[:invite_code] if resource.invite_code.blank?
+    resource.invite_code            = @invite&.code if resource.invite_code.blank?
     resource.registration_form_time = session[:registration_form_time]
     resource.sign_up_ip             = request.remote_ip
 

app/controllers/backups_controller.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class BackupsController < ApplicationController
+  include RoutingHelper
+
+  skip_before_action :require_functional!
+
+  before_action :authenticate_user!
+  before_action :set_backup
+
+  def download
+    case Paperclip::Attachment.default_options[:storage]
+    when :s3
+      redirect_to @backup.dump.expiring_url(10)
+    when :fog
+      if Paperclip::Attachment.default_options.dig(:fog_credentials, :openstack_temp_url_key).present?
+        redirect_to @backup.dump.expiring_url(Time.now.utc + 10)
+      else
+        redirect_to full_asset_url(@backup.dump.url)
+      end
+    when :filesystem
+      redirect_to full_asset_url(@backup.dump.url)
+    end
+  end
+
+  private
+
+  def set_backup
+    @backup = current_user.backups.find(params[:id])
+  end
+end

app/controllers/media_controller.rb

@@ -46,6 +46,6 @@ class MediaController < ApplicationController
   end
 
   def allow_iframing
-    response.headers['X-Frame-Options'] = 'ALLOWALL'
+    response.headers.delete('X-Frame-Options')
   end
 end

app/controllers/oauth/authorized_applications_controller.rb

@@ -8,6 +8,8 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
   before_action :require_not_suspended!, only: :destroy
   before_action :set_body_classes
 
+  before_action :set_last_used_at_by_app, only: :index, unless: -> { request.format == :json }
+
   skip_before_action :require_functional!
 
   include Localized
@@ -30,4 +32,14 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
   def require_not_suspended!
     forbidden if current_account.suspended?
   end
+
+  def set_last_used_at_by_app
+    @last_used_at_by_app = Doorkeeper::AccessToken
+                           .select('DISTINCT ON (application_id) application_id, last_used_at')
+                           .where(resource_owner_id: current_resource_owner.id)
+                           .where.not(last_used_at: nil)
+                           .order(application_id: :desc, last_used_at: :desc)
+                           .pluck(:application_id, :last_used_at)
+                           .to_h
+  end
 end

app/controllers/relationships_controller.rb

@@ -19,6 +19,8 @@ class RelationshipsController < ApplicationController
     @form.save
   rescue ActionController::ParameterMissing
     # Do nothing
+  rescue Mastodon::NotPermittedError, ActiveRecord::RecordNotFound
+    flash[:alert] = I18n.t('relationships.follow_failure') if action_from_button == 'follow'
   ensure
     redirect_to relationships_path(filter_params)
   end
@@ -60,8 +62,8 @@ class RelationshipsController < ApplicationController
       'unfollow'
     elsif params[:remove_from_followers]
       'remove_from_followers'
-    elsif params[:block_domains]
-      'block_domains'
+    elsif params[:block_domains] || params[:remove_domains_from_followers]
+      'remove_domains_from_followers'
     end
   end
 

app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb

@@ -52,7 +52,7 @@ module Settings
             end
           else
             flash[:error] = I18n.t('webauthn_credentials.create.error')
-            status = :internal_server_error
+            status = :unprocessable_entity
           end
         else
           flash[:error] = t('webauthn_credentials.create.error')

app/controllers/statuses_controller.rb

@@ -43,7 +43,7 @@ class StatusesController < ApplicationController
     return not_found if @status.hidden? || @status.reblog?
 
     expires_in 180, public: true
-    response.headers['X-Frame-Options'] = 'ALLOWALL'
+    response.headers.delete('X-Frame-Options')
 
     render layout: 'embedded'
   end

app/controllers/well_known/webfinger_controller.rb

@@ -18,7 +18,14 @@ module WellKnown
     private
 
     def set_account
-      @account = Account.find_local!(username_from_resource)
+      username = username_from_resource
+      @account = begin
+        if username == Rails.configuration.x.local_domain
+          Account.representative
+        else
+          Account.find_local!(username)
+        end
+      end
     end
 
     def username_from_resource

app/helpers/formatting_helper.rb

@@ -58,6 +58,10 @@ module FormattingHelper
   end
 
   def account_field_value_format(field, with_rel_me: true)
-    html_aware_format(field.value, field.account.local?, with_rel_me: with_rel_me, with_domains: true, multiline: false)
+    if field.verified? && !field.account.local?
+      TextFormatter.shortened_link(field.value_for_verification)
+    else
+      html_aware_format(field.value, field.account.local?, with_rel_me: with_rel_me, with_domains: true, multiline: false)
+    end
   end
 end

app/javascript/mastodon/actions/compose.js

@@ -165,11 +165,19 @@ export function submitCompose(routerHistory) {
     // API call.
     let media_attributes;
     if (statusId !== null) {
-      media_attributes = media.map(item => ({
-        id: item.get('id'),
-        description: item.get('description'),
-        focus: item.get('focus'),
-      }));
+      media_attributes = media.map(item => {
+        let focus;
+
+        if (item.getIn(['meta', 'focus'])) {
+          focus = `${item.getIn(['meta', 'focus', 'x']).toFixed(2)},${item.getIn(['meta', 'focus', 'y']).toFixed(2)}`;
+        }
+
+        return {
+          id: item.get('id'),
+          description: item.get('description'),
+          focus,
+        };
+      });
     }
 
     api(getState).request({

app/javascript/mastodon/components/column_back_button.js

@@ -15,10 +15,10 @@ export default class ColumnBackButton extends React.PureComponent {
   };
 
   handleClick = () => {
-    if (window.history && window.history.length === 1) {
-      this.context.router.history.push('/');
-    } else {
+    if (window.history && window.history.state) {
       this.context.router.history.goBack();
+    } else {
+      this.context.router.history.push('/');
     }
   };
 

app/javascript/mastodon/components/column_header.js

@@ -43,14 +43,6 @@ class ColumnHeader extends React.PureComponent {
     animating: false,
   };
 
-  historyBack = () => {
-    if (window.history && window.history.length === 1) {
-      this.context.router.history.push('/');
-    } else {
-      this.context.router.history.goBack();
-    }
-  };
-
   handleToggleClick = (e) => {
     e.stopPropagation();
     this.setState({ collapsed: !this.state.collapsed, animating: true });
@@ -69,7 +61,11 @@ class ColumnHeader extends React.PureComponent {
   };
 
   handleBackClick = () => {
-    this.historyBack();
+    if (window.history && window.history.state) {
+      this.context.router.history.goBack();
+    } else {
+      this.context.router.history.push('/');
+    }
   };
 
   handleTransitionEnd = () => {

app/javascript/mastodon/containers/status_container.js

@@ -56,6 +56,8 @@ const messages = defineMessages({
   redraftMessage: { id: 'confirmations.redraft.message', defaultMessage: 'Are you sure you want to delete this status and re-draft it? Favourites and boosts will be lost, and replies to the original post will be orphaned.' },
   replyConfirm: { id: 'confirmations.reply.confirm', defaultMessage: 'Reply' },
   replyMessage: { id: 'confirmations.reply.message', defaultMessage: 'Replying now will overwrite the message you are currently composing. Are you sure you want to proceed?' },
+  editConfirm: { id: 'confirmations.edit.confirm', defaultMessage: 'Edit' },
+  editMessage: { id: 'confirmations.edit.message', defaultMessage: 'Editing now will overwrite the message you are currently composing. Are you sure you want to proceed?' },
   blockDomainConfirm: { id: 'confirmations.domain_block.confirm', defaultMessage: 'Hide entire domain' },
 });
 
@@ -149,7 +151,18 @@ const mapDispatchToProps = (dispatch, { intl, contextType }) => ({
   },
 
   onEdit (status, history) {
-    dispatch(editStatus(status.get('id'), history));
+    dispatch((_, getState) => {
+      let state = getState();
+      if (state.getIn(['compose', 'text']).trim().length !== 0) {
+        dispatch(openModal('CONFIRM', {
+          message: intl.formatMessage(messages.editMessage),
+          confirm: intl.formatMessage(messages.editConfirm),
+          onConfirm: () => dispatch(editStatus(status.get('id'), history)),
+        }));
+      } else {
+        dispatch(editStatus(status.get('id'), history));
+      }
+    });
   },
 
   onTranslate (status) {

app/javascript/mastodon/features/compose/components/language_dropdown.js

@@ -210,7 +210,7 @@ class LanguageDropdownMenu extends React.PureComponent {
 
     return (
       <div key={lang[0]} role='option' tabIndex='0' data-index={lang[0]} className={classNames('language-dropdown__dropdown__results__item', { active: lang[0] === value })} aria-selected={lang[0] === value} onClick={this.handleClick} onKeyDown={this.handleKeyDown}>
-        <span className='language-dropdown__dropdown__results__item__native-name'>{lang[2]}</span> <span className='language-dropdown__dropdown__results__item__common-name'>({lang[1]})</span>
+        <span className='language-dropdown__dropdown__results__item__native-name' lang={lang[0]}>{lang[2]}</span> <span className='language-dropdown__dropdown__results__item__common-name'>({lang[1]})</span>
       </div>
     );
   };

app/javascript/mastodon/features/ui/components/header.js

@@ -22,8 +22,8 @@ const mapDispatchToProps = (dispatch) => ({
   },
 });
 
-export default @connect(null, mapDispatchToProps)
-@withRouter
+export default @withRouter
+@connect(null, mapDispatchToProps)
 class Header extends React.PureComponent {
 
   static contextTypes = {

app/javascript/mastodon/features/ui/components/navigation_panel.js

@@ -82,8 +82,8 @@ class NavigationPanel extends React.Component {
         {signedIn && (
           <React.Fragment>
             <ColumnLink transparent to='/conversations' icon='at' text={intl.formatMessage(messages.direct)} />
-            <ColumnLink transparent to='/favourites' icon='star' text={intl.formatMessage(messages.favourites)} />
             <ColumnLink transparent to='/bookmarks' icon='bookmark' text={intl.formatMessage(messages.bookmarks)} />
+            <ColumnLink transparent to='/favourites' icon='star' text={intl.formatMessage(messages.favourites)} />
             <ColumnLink transparent to='/lists' icon='list-ul' text={intl.formatMessage(messages.lists)} />
 
             <ListPanel />

app/javascript/mastodon/features/ui/index.js

@@ -474,10 +474,10 @@ class UI extends React.PureComponent {
   };
 
   handleHotkeyBack = () => {
-    if (window.history && window.history.length === 1) {
-      this.context.router.history.push('/');
-    } else {
+    if (window.history && window.history.state) {
       this.context.router.history.goBack();
+    } else {
+      this.context.router.history.push('/');
     }
   };
 

app/javascript/mastodon/locales/en.json

@@ -162,6 +162,8 @@
   "confirmations.discard_edit_media.message": "You have unsaved changes to the media description or preview, discard them anyway?",
   "confirmations.domain_block.confirm": "Block entire domain",
   "confirmations.domain_block.message": "Are you really, really sure you want to block the entire {domain}? In most cases a few targeted blocks or mutes are sufficient and preferable. You will not see content from that domain in any public timelines or your notifications. Your followers from that domain will be removed.",
+  "confirmations.edit.confirm": "Edit",
+  "confirmations.edit.message": "Editing now will overwrite the message you are currently composing. Are you sure you want to proceed?",
   "confirmations.logout.confirm": "Log out",
   "confirmations.logout.message": "Are you sure you want to log out?",
   "confirmations.mute.confirm": "Mute",

app/javascript/mastodon/reducers/compose.js

@@ -186,11 +186,12 @@ const ignoreSuggestion = (state, position, token, completion, path) => {
 };
 
 const sortHashtagsByUse = (state, tags) => {
-  const personalHistory = state.get('tagHistory');
+  const personalHistory = state.get('tagHistory').map(tag => tag.toLowerCase());
 
-  return tags.sort((a, b) => {
-    const usedA = personalHistory.includes(a.name);
-    const usedB = personalHistory.includes(b.name);
+  const tagsWithLowercase = tags.map(t => ({ ...t, lowerName: t.name.toLowerCase() }));
+  const sorted = tagsWithLowercase.sort((a, b) => {
+    const usedA = personalHistory.includes(a.lowerName);
+    const usedB = personalHistory.includes(b.lowerName);
 
     if (usedA === usedB) {
       return 0;
@@ -200,6 +201,8 @@ const sortHashtagsByUse = (state, tags) => {
       return 1;
     }
   });
+  sorted.forEach(tag => delete tag.lowerName);
+  return sorted;
 };
 
 const insertEmoji = (state, position, emojiData, needsSpace) => {

app/javascript/styles/mastodon-light/diff.scss

@@ -254,6 +254,10 @@ html {
   border-color: $ui-base-color;
 }
 
+.upload-progress__backdrop {
+  background: $ui-base-color;
+}
+
 // Change the background colors of statuses
 .focusable:focus {
   background: $ui-base-color;

app/javascript/styles/mastodon/admin.scss

@@ -384,7 +384,7 @@ $content-width: 840px;
           position: fixed;
           z-index: 10;
           width: 100%;
-          height: calc(100vh - 56px);
+          height: calc(100% - 56px);
           left: 0;
           bottom: 0;
           overflow-y: auto;

app/javascript/styles/mastodon/components.scss

@@ -4482,6 +4482,7 @@ a.status-card.compact:hover {
   display: flex;
   align-items: center;
   justify-content: center;
+  text-align: center;
   color: $secondary-text-color;
   font-size: 18px;
   font-weight: 500;
@@ -4516,7 +4517,7 @@ a.status-card.compact:hover {
   width: 100%;
   height: 6px;
   border-radius: 6px;
-  background: $ui-base-lighter-color;
+  background: darken($simple-background-color, 8%);
   position: relative;
   margin-top: 5px;
 }

app/lib/account_reach_finder.rb

@@ -6,7 +6,7 @@ class AccountReachFinder
   end
 
   def inboxes
-    (followers_inboxes + reporters_inboxes + relay_inboxes).uniq
+    (followers_inboxes + reporters_inboxes + recently_mentioned_inboxes + relay_inboxes).uniq
   end
 
   private
@@ -19,6 +19,13 @@ class AccountReachFinder
     Account.where(id: @account.targeted_reports.select(:account_id)).inboxes
   end
 
+  def recently_mentioned_inboxes
+    cutoff_id       = Mastodon::Snowflake.id_at(2.days.ago, with_random: false)
+    recent_statuses = @account.statuses.recent.where(id: cutoff_id...).limit(200)
+
+    Account.joins(:mentions).where(mentions: { status: recent_statuses }).inboxes.take(2000)
+  end
+
   def relay_inboxes
     Relay.enabled.pluck(:inbox_url)
   end

app/lib/admin/system_check.rb

@@ -2,6 +2,7 @@
 
 class Admin::SystemCheck
   ACTIVE_CHECKS = [
+    Admin::SystemCheck::MediaPrivacyCheck,
     Admin::SystemCheck::DatabaseSchemaCheck,
     Admin::SystemCheck::SidekiqProcessCheck,
     Admin::SystemCheck::RulesCheck,

app/lib/admin/system_check/elasticsearch_check.rb

@@ -31,7 +31,7 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck
   def running_version
     @running_version ||= begin
       Chewy.client.info['version']['number']
-    rescue Faraday::ConnectionFailed
+    rescue Faraday::ConnectionFailed, Elasticsearch::Transport::Transport::Error
       nil
     end
   end

app/lib/admin/system_check/media_privacy_check.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+class Admin::SystemCheck::MediaPrivacyCheck < Admin::SystemCheck::BaseCheck
+  include RoutingHelper
+
+  def skip?
+    !current_user.can?(:view_devops)
+  end
+
+  def pass?
+    check_media_uploads!
+    @failure_message.nil?
+  end
+
+  def message
+    Admin::SystemCheck::Message.new(@failure_message, @failure_value, @failure_action, true)
+  end
+
+  private
+
+  def check_media_uploads!
+    if Rails.configuration.x.use_s3
+      check_media_listing_inaccessible_s3!
+    else
+      check_media_listing_inaccessible!
+    end
+  end
+
+  def check_media_listing_inaccessible!
+    full_url = full_asset_url(media_attachment.file.url(:original, false))
+
+    # Check if we can list the uploaded file. If true, that's an error
+    directory_url = Addressable::URI.parse(full_url)
+    directory_url.query = nil
+    filename = directory_url.path.gsub(%r{.*/}, '')
+    directory_url.path = directory_url.path.gsub(%r{/[^/]+\Z}, '/')
+    Request.new(:get, directory_url, allow_local: true).perform do |res|
+      if res.truncated_body&.include?(filename)
+        @failure_message = use_storage? ? :upload_check_privacy_error_object_storage : :upload_check_privacy_error
+        @failure_action = 'https://docs.joinmastodon.org/admin/optional/object-storage/#FS'
+      end
+    end
+  rescue
+    nil
+  end
+
+  def check_media_listing_inaccessible_s3!
+    urls_to_check = []
+    paperclip_options = Paperclip::Attachment.default_options
+    s3_protocol = paperclip_options[:s3_protocol]
+    s3_host_alias = paperclip_options[:s3_host_alias]
+    s3_host_name  = paperclip_options[:s3_host_name]
+    bucket_name = paperclip_options.dig(:s3_credentials, :bucket)
+
+    urls_to_check << "#{s3_protocol}://#{s3_host_alias}/" if s3_host_alias.present?
+    urls_to_check << "#{s3_protocol}://#{s3_host_name}/#{bucket_name}/"
+    urls_to_check.uniq.each do |full_url|
+      check_s3_listing!(full_url)
+      break if @failure_message.present?
+    end
+  rescue
+    nil
+  end
+
+  def check_s3_listing!(full_url)
+    bucket_url = Addressable::URI.parse(full_url)
+    bucket_url.path = bucket_url.path.delete_suffix(media_attachment.file.path(:original))
+    bucket_url.query = "max-keys=1&x-random=#{SecureRandom.hex(10)}"
+    Request.new(:get, bucket_url, allow_local: true).perform do |res|
+      if res.truncated_body&.include?('ListBucketResult')
+        @failure_message = :upload_check_privacy_error_object_storage
+        @failure_action  = 'https://docs.joinmastodon.org/admin/optional/object-storage/#S3'
+      end
+    end
+  end
+
+  def media_attachment
+    @media_attachment ||= begin
+      attachment = Account.representative.media_attachments.first
+      if attachment.present?
+        attachment.touch # rubocop:disable Rails/SkipsModelValidations
+        attachment
+      else
+        create_test_attachment!
+      end
+    end
+  end
+
+  def create_test_attachment!
+    Tempfile.create(%w(test-upload .jpg), binmode: true) do |tmp_file|
+      tmp_file.write(
+        Base64.decode64(
+          '/9j/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAYAAAA' \
+          'AAAD/2wCEAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBA' \
+          'QEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE' \
+          'BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAf/AABEIAAEAAgMBEQACEQEDEQH/x' \
+          'ABKAAEAAAAAAAAAAAAAAAAAAAALEAEAAAAAAAAAAAAAAAAAAAAAAQEAAAAAAAAAAAAAAAA' \
+          'AAAAAEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwA/8H//2Q=='
+        )
+      )
+      tmp_file.flush
+      Account.representative.media_attachments.create!(file: tmp_file)
+    end
+  end
+end

app/lib/admin/system_check/message.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
 
 class Admin::SystemCheck::Message
-  attr_reader :key, :value, :action
+  attr_reader :key, :value, :action, :critical
 
-  def initialize(key, value = nil, action = nil)
-    @key    = key
-    @value  = value
-    @action = action
+  def initialize(key, value = nil, action = nil, critical = false)
+    @key      = key
+    @value    = value
+    @action   = action
+    @critical = critical
   end
 end

app/lib/application_extension.rb

@@ -9,10 +9,6 @@ module ApplicationExtension
     validates :redirect_uri, length: { maximum: 2_000 }
   end
 
-  def most_recently_used_access_token
-    @most_recently_used_access_token ||= access_tokens.where.not(last_used_at: nil).order(last_used_at: :desc).first
-  end
-
   def confirmation_redirect_uri
     redirect_uri.lines.first.strip
   end

app/lib/link_details_extractor.rb

@@ -140,7 +140,7 @@ class LinkDetailsExtractor
   end
 
   def html
-    player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
+    player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowfullscreen: 'true', allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
   end
 
   def width

app/lib/plain_text_formatter.rb

@@ -18,7 +18,7 @@ class PlainTextFormatter
     if local?
       text
     else
-      strip_tags(insert_newlines).chomp
+      html_entities.decode(strip_tags(insert_newlines)).chomp
     end
   end
 
@@ -27,4 +27,8 @@ class PlainTextFormatter
   def insert_newlines
     text.gsub(NEWLINE_TAGS_RE) { |match| "#{match}\n" }
   end
+
+  def html_entities
+    HTMLEntities.new
+  end
 end

app/lib/request.rb

@@ -7,11 +7,48 @@ require 'resolv'
 # Monkey-patch the HTTP.rb timeout class to avoid using a timeout block
 # around the Socket#open method, since we use our own timeout blocks inside
 # that method
+#
+# Also changes how the read timeout behaves so that it is cumulative (closer
+# to HTTP::Timeout::Global, but still having distinct timeouts for other
+# operation types)
 class HTTP::Timeout::PerOperation
   def connect(socket_class, host, port, nodelay = false)
     @socket = socket_class.open(host, port)
     @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay
   end
+
+  # Reset deadline when the connection is re-used for different requests
+  def reset_counter
+    @deadline = nil
+  end
+
+  # Read data from the socket
+  def readpartial(size, buffer = nil)
+    @deadline ||= Process.clock_gettime(Process::CLOCK_MONOTONIC) + @read_timeout
+
+    timeout = false
+    loop do
+      result = @socket.read_nonblock(size, buffer, exception: false)
+
+      return :eof if result.nil?
+
+      remaining_time = @deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      raise HTTP::TimeoutError, "Read timed out after #{@read_timeout} seconds" if timeout || remaining_time <= 0
+      return result if result != :wait_readable
+
+      # marking the socket for timeout. Why is this not being raised immediately?
+      # it seems there is some race-condition on the network level between calling
+      # #read_nonblock and #wait_readable, in which #read_nonblock signalizes waiting
+      # for reads, and when waiting for x seconds, it returns nil suddenly without completing
+      # the x seconds. In a normal case this would be a timeout on wait/read, but it can
+      # also mean that the socket has been closed by the server. Therefore we "mark" the
+      # socket for timeout and try to read more bytes. If it returns :eof, it's all good, no
+      # timeout. Else, the first timeout was a proper timeout.
+      # This hack has to be done because io/wait#wait_readable doesn't provide a value for when
+      # the socket is closed by the server, and HTTP::Parser doesn't provide the limit for the chunks.
+      timeout = true unless @socket.to_io.wait_readable(remaining_time)
+    end
+  end
 end
 
 class Request

app/lib/scope_parser.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class ScopeParser < Parslet::Parser
-  rule(:term)      { match('[a-z]').repeat(1).as(:term) }
+  rule(:term)      { match('[a-z_]').repeat(1).as(:term) }
   rule(:colon)     { str(':') }
   rule(:access)    { (str('write') | str('read')).as(:access) }
   rule(:namespace) { str('admin').as(:namespace) }

app/lib/text_formatter.rb

@@ -48,6 +48,26 @@ class TextFormatter
     html.html_safe # rubocop:disable Rails/OutputSafety
   end
 
+  class << self
+    include ERB::Util
+
+    def shortened_link(url, rel_me: false)
+      url = Addressable::URI.parse(url).to_s
+      rel = rel_me ? (DEFAULT_REL + %w(me)) : DEFAULT_REL
+
+      prefix      = url.match(URL_PREFIX_REGEX).to_s
+      display_url = url[prefix.length, 30]
+      suffix      = url[prefix.length + 30..-1]
+      cutoff      = url[prefix.length..-1].length > 30
+
+      <<~HTML.squish.html_safe # rubocop:disable Rails/OutputSafety
+        <a href="#{h(url)}" target="_blank" rel="#{rel.join(' ')}"><span class="invisible">#{h(prefix)}</span><span class="#{cutoff ? 'ellipsis' : ''}">#{h(display_url)}</span><span class="invisible">#{h(suffix)}</span></a>
+      HTML
+    rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
+      h(url)
+    end
+  end
+
   private
 
   def rewrite
@@ -70,19 +90,7 @@ class TextFormatter
   end
 
   def link_to_url(entity)
-    url = Addressable::URI.parse(entity[:url]).to_s
-    rel = with_rel_me? ? (DEFAULT_REL + %w(me)) : DEFAULT_REL
-
-    prefix      = url.match(URL_PREFIX_REGEX).to_s
-    display_url = url[prefix.length, 30]
-    suffix      = url[prefix.length + 30..-1]
-    cutoff      = url[prefix.length..-1].length > 30
-
-    <<~HTML.squish
-      <a href="#{h(url)}" target="_blank" rel="#{rel.join(' ')}"><span class="invisible">#{h(prefix)}</span><span class="#{cutoff ? 'ellipsis' : ''}">#{h(display_url)}</span><span class="invisible">#{h(suffix)}</span></a>
-    HTML
-  rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
-    h(entity[:url])
+    TextFormatter.shortened_link(entity[:url], rel_me: with_rel_me?)
   end
 
   def link_to_hashtag(entity)

app/lib/vacuum/access_tokens_vacuum.rb

@@ -9,10 +9,12 @@ class Vacuum::AccessTokensVacuum
   private
 
   def vacuum_revoked_access_tokens!
-    Doorkeeper::AccessToken.where.not(revoked_at: nil).where('revoked_at < NOW()').delete_all
+    Doorkeeper::AccessToken.where.not(expires_in: nil).where('created_at + make_interval(secs => expires_in) < NOW()').in_batches.delete_all
+    Doorkeeper::AccessToken.where.not(revoked_at: nil).where('revoked_at < NOW()').in_batches.delete_all
   end
 
   def vacuum_revoked_access_grants!
-    Doorkeeper::AccessGrant.where.not(revoked_at: nil).where('revoked_at < NOW()').delete_all
+    Doorkeeper::AccessGrant.where.not(expires_in: nil).where('created_at + make_interval(secs => expires_in) < NOW()').in_batches.delete_all
+    Doorkeeper::AccessGrant.where.not(revoked_at: nil).where('revoked_at < NOW()').in_batches.delete_all
   end
 end

app/mailers/application_mailer.rb

@@ -7,6 +7,8 @@ class ApplicationMailer < ActionMailer::Base
   helper :instance
   helper :formatting
 
+  after_action :set_autoreply_headers!
+
   protected
 
   def locale_for_account(account)
@@ -14,4 +16,10 @@ class ApplicationMailer < ActionMailer::Base
       yield
     end
   end
+
+  def set_autoreply_headers!
+    headers['Precedence'] = 'list'
+    headers['X-Auto-Response-Suppress'] = 'All'
+    headers['Auto-Submitted'] = 'auto-generated'
+  end
 end

app/models/account.rb

@@ -107,7 +107,7 @@ class Account < ApplicationRecord
   scope :bots, -> { where(actor_type: %w(Application Service)) }
   scope :groups, -> { where(actor_type: 'Group') }
   scope :alphabetic, -> { order(domain: :asc, username: :asc) }
-  scope :matches_username, ->(value) { where(arel_table[:username].matches("#{value}%")) }
+  scope :matches_username, ->(value) { where('lower((username)::text) LIKE lower(?)', "#{value}%") }
   scope :matches_display_name, ->(value) { where(arel_table[:display_name].matches("#{value}%")) }
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
   scope :without_unapproved, -> { left_outer_joins(:user).remote.or(left_outer_joins(:user).merge(User.approved.confirmed)) }

app/models/account_conversation.rb

@@ -16,34 +16,44 @@
 class AccountConversation < ApplicationRecord
   include Redisable
 
+  attr_writer :participant_accounts
+
+  before_validation :set_last_status
   after_commit :push_to_streaming_api
 
   belongs_to :account
   belongs_to :conversation
   belongs_to :last_status, class_name: 'Status'
 
-  before_validation :set_last_status
-
   def participant_account_ids=(arr)
     self[:participant_account_ids] = arr.sort
+    @participant_accounts = nil
   end
 
   def participant_accounts
-    if participant_account_ids.empty?
-      [account]
-    else
-      participants = Account.where(id: participant_account_ids)
-      participants.empty? ? [account] : participants
-    end
+    @participant_accounts ||= Account.where(id: participant_account_ids).to_a
+    @participant_accounts.presence || [account]
   end
 
   class << self
     def to_a_paginated_by_id(limit, options = {})
-      if options[:min_id]
-        paginate_by_min_id(limit, options[:min_id], options[:max_id]).reverse
-      else
-        paginate_by_max_id(limit, options[:max_id], options[:since_id]).to_a
+      array = begin
+        if options[:min_id]
+          paginate_by_min_id(limit, options[:min_id], options[:max_id]).reverse
+        else
+          paginate_by_max_id(limit, options[:max_id], options[:since_id]).to_a
+        end
       end
+
+      # Preload participants
+      participant_ids = array.flat_map(&:participant_account_ids)
+      accounts_by_id = Account.where(id: participant_ids).index_by(&:id)
+
+      array.each do |conversation|
+        conversation.participant_accounts = conversation.participant_account_ids.filter_map { |id| accounts_by_id[id] }
+      end
+
+      array
     end
 
     def paginate_by_min_id(limit, min_id = nil, max_id = nil)

app/models/backup.rb

@@ -17,6 +17,6 @@
 class Backup < ApplicationRecord
   belongs_to :user, inverse_of: :backups
 
-  has_attached_file :dump
+  has_attached_file :dump, s3_permissions: ->(*) { ENV['S3_PERMISSION'] == '' ? nil : 'private' }
   do_not_validate_attachment_file_type :dump
 end

app/models/concerns/attachmentable.rb

@@ -22,15 +22,14 @@ module Attachmentable
 
   included do
     def self.has_attached_file(name, options = {}) # rubocop:disable Naming/PredicateName
-      options = { validate_media_type: false }.merge(options)
       super(name, options)
-      send(:"before_#{name}_post_process") do
+
+      send(:"before_#{name}_validate", prepend: true) do
         attachment = send(name)
         check_image_dimension(attachment)
         set_file_content_type(attachment)
         obfuscate_file_name(attachment)
         set_file_extension(attachment)
-        Paperclip::Validators::MediaTypeSpoofDetectionValidator.new(attributes: [name]).validate(self)
       end
     end
   end

app/models/concerns/ldap_authenticable.rb

@@ -6,7 +6,7 @@ module LdapAuthenticable
   class_methods do
     def authenticate_with_ldap(params = {})
       ldap   = Net::LDAP.new(ldap_options)
-      filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, mail: Devise.ldap_mail, email: params[:email])
+      filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, mail: Devise.ldap_mail, email: Net::LDAP::Filter.escape(params[:email]))
 
       if (user_info = ldap.bind_as(base: Devise.ldap_base, filter: filter, password: params[:password]))
         ldap_get_user(user_info.first)

app/models/form/account_batch.rb

@@ -17,8 +17,8 @@ class Form::AccountBatch
       unfollow!
     when 'remove_from_followers'
       remove_from_followers!
-    when 'block_domains'
-      block_domains!
+    when 'remove_domains_from_followers'
+      remove_domains_from_followers!
     when 'approve'
       approve!
     when 'reject'
@@ -35,9 +35,15 @@ class Form::AccountBatch
   private
 
   def follow!
+    error = nil
+
     accounts.each do |target_account|
       FollowService.new.call(current_account, target_account)
+    rescue Mastodon::NotPermittedError, ActiveRecord::RecordNotFound => e
+      error ||= e
     end
+
+    raise error if error.present?
   end
 
   def unfollow!
@@ -50,10 +56,8 @@ class Form::AccountBatch
     RemoveFromFollowersService.new.call(current_account, account_ids)
   end
 
-  def block_domains!
-    AfterAccountDomainBlockWorker.push_bulk(account_domains) do |domain|
-      [current_account.id, domain]
-    end
+  def remove_domains_from_followers!
+    RemoveDomainsFromFollowersService.new.call(current_account, account_domains)
   end
 
   def account_domains
@@ -119,7 +123,18 @@ class Form::AccountBatch
       account: current_account,
       action: :suspend
     )
+
     Admin::SuspensionWorker.perform_async(account.id)
+
+    # Suspending a single account closes their associated reports, so
+    # mass-suspending would be consistent.
+    Report.where(target_account: account).unresolved.find_each do |report|
+      authorize(report, :update?)
+      log_action(:resolve, report)
+      report.resolve!(current_account)
+    rescue Mastodon::NotPermittedError
+      # This should not happen, but just in case, do not fail early
+    end
   end
 
   def approve_account(account)

app/models/identity.rb

@@ -12,7 +12,7 @@
 #
 
 class Identity < ApplicationRecord
-  belongs_to :user, dependent: :destroy
+  belongs_to :user
   validates :uid, presence: true, uniqueness: { scope: :provider }
   validates :provider, presence: true
 

app/models/user.rb

@@ -504,11 +504,14 @@ class User < ApplicationRecord
   def prepare_new_user!
     BootstrapTimelineWorker.perform_async(account_id)
     ActivityTracker.increment('activity:accounts:local')
+    ActivityTracker.record('activity:logins', id)
     UserMailer.welcome(self).deliver_later
     TriggerWebhookWorker.perform_async('account.approved', 'Account', account_id)
   end
 
   def prepare_returning_user!
+    return unless confirmed?
+
     ActivityTracker.record('activity:logins', id)
     regenerate_feed! if needs_feed_update?
   end

app/models/webhook.rb

@@ -20,6 +20,8 @@ class Webhook < ApplicationRecord
     report.created
   ).freeze
 
+  attr_writer :current_account
+
   scope :enabled, -> { where(enabled: true) }
 
   validates :url, presence: true, url: true
@@ -27,6 +29,7 @@ class Webhook < ApplicationRecord
   validates :events, presence: true
 
   validate :validate_events
+  validate :validate_permissions
 
   before_validation :strip_events
   before_validation :generate_secret
@@ -43,12 +46,29 @@ class Webhook < ApplicationRecord
     update!(enabled: false)
   end
 
+  def required_permissions
+    events.map { |event| Webhook.permission_for_event(event) }
+  end
+
+  def self.permission_for_event(event)
+    case event
+    when 'account.approved', 'account.created', 'account.updated'
+      :manage_users
+    when 'report.created'
+      :manage_reports
+    end
+  end
+
   private
 
   def validate_events
     errors.add(:events, :invalid) if events.any? { |e| !EVENTS.include?(e) }
   end
 
+  def validate_permissions
+    errors.add(:events, :invalid_permissions) if defined?(@current_account) && required_permissions.any? { |permission| !@current_account.user_role.can?(permission) }
+  end
+
   def strip_events
     self.events = events.map { |str| str.strip.presence }.compact if events.present?
   end

app/policies/webhook_policy.rb

@@ -14,7 +14,7 @@ class WebhookPolicy < ApplicationPolicy
   end
 
   def update?
-    role.can?(:manage_webhooks)
+    role.can?(:manage_webhooks) && record.required_permissions.all? { |permission| role.can?(permission) }
   end
 
   def enable?
@@ -30,6 +30,6 @@ class WebhookPolicy < ApplicationPolicy
   end
 
   def destroy?
-    role.can?(:manage_webhooks)
+    role.can?(:manage_webhooks) && record.required_permissions.all? { |permission| role.can?(permission) }
   end
 end

app/serializers/rest/preview_card_serializer.rb

@@ -11,4 +11,8 @@ class REST::PreviewCardSerializer < ActiveModel::Serializer
   def image
     object.image? ? full_asset_url(object.image.url(:original)) : nil
   end
+
+  def html
+    Sanitize.fragment(object.html, Sanitize::Config::MASTODON_OEMBED)
+  end
 end

app/services/follow_migration_service.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class FollowMigrationService < FollowService
+  # Follow an account with the same settings as another account, and unfollow the old account once the request is sent
+  # @param [Account] source_account From which to follow
+  # @param [Account] target_account Account to follow
+  # @param [Account] old_target_account Account to unfollow once the follow request has been sent to the new one
+  # @option [Boolean] bypass_locked Whether to immediately follow the new account even if it is locked
+  def call(source_account, target_account, old_target_account, bypass_locked: false)
+    @old_target_account = old_target_account
+
+    follow    = source_account.active_relationships.find_by(target_account: old_target_account)
+    reblogs   = follow&.show_reblogs?
+    notify    = follow&.notify?
+    languages = follow&.languages
+
+    super(source_account, target_account, reblogs: reblogs, notify: notify, languages: languages, bypass_locked: bypass_locked, bypass_limit: true)
+  end
+
+  private
+
+  def request_follow!
+    follow_request = @source_account.request_follow!(@target_account, **follow_options.merge(rate_limit: @options[:with_rate_limit], bypass_limit: @options[:bypass_limit]))
+
+    if @target_account.local?
+      LocalNotificationWorker.perform_async(@target_account.id, follow_request.id, follow_request.class.name, 'follow_request')
+      UnfollowService.new.call(@source_account, @old_target_account, skip_unmerge: true)
+    elsif @target_account.activitypub?
+      ActivityPub::MigratedFollowDeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url, @old_target_account.id)
+    end
+
+    follow_request
+  end
+
+  def direct_follow!
+    follow = super
+    UnfollowService.new.call(@source_account, @old_target_account, skip_unmerge: true)
+    follow
+  end
+end

app/services/remove_domains_from_followers_service.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class RemoveDomainsFromFollowersService < BaseService
+  include Payloadable
+
+  def call(source_account, target_domains)
+    source_account.passive_relationships.where(account_id: Account.where(domain: target_domains)).find_each do |follow|
+      follow.destroy
+
+      create_notification(follow) if source_account.local? && !follow.account.local? && follow.account.activitypub?
+    end
+  end
+
+  private
+
+  def create_notification(follow)
+    ActivityPub::DeliveryWorker.perform_async(build_json(follow), follow.target_account_id, follow.account.inbox_url)
+  end
+
+  def build_json(follow)
+    Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer))
+  end
+end

app/services/remove_status_service.rb

@@ -12,6 +12,7 @@ class RemoveStatusService < BaseService
   # @option  [Boolean] :immediate
   # @option  [Boolean] :preserve
   # @option  [Boolean] :original_removed
+  # @option  [Boolean] :skip_streaming
   def call(status, **options)
     @payload  = Oj.dump(event: :delete, payload: status.id.to_s)
     @status   = status
@@ -52,6 +53,9 @@ class RemoveStatusService < BaseService
 
   private
 
+  # The following FeedManager calls all do not result in redis publishes for
+  # streaming, as the `:update` option is false
+
   def remove_from_self
     FeedManager.instance.unpush_from_home(@account, @status)
   end
@@ -75,6 +79,8 @@ class RemoveStatusService < BaseService
     # followers. Here we send a delete to actively mentioned accounts
     # that may not follow the account
 
+    return if skip_streaming?
+
     @status.active_mentions.find_each do |mention|
       redis.publish("timeline:#{mention.account_id}", @payload)
     end
@@ -103,7 +109,7 @@ class RemoveStatusService < BaseService
     # without us being able to do all the fancy stuff
 
     @status.reblogs.rewhere(deleted_at: [nil, @status.deleted_at]).includes(:account).reorder(nil).find_each do |reblog|
-      RemoveStatusService.new.call(reblog, original_removed: true)
+      RemoveStatusService.new.call(reblog, original_removed: true, skip_streaming: skip_streaming?)
     end
   end
 
@@ -114,6 +120,8 @@ class RemoveStatusService < BaseService
 
     return unless @status.public_visibility?
 
+    return if skip_streaming?
+
     @status.tags.map(&:name).each do |hashtag|
       redis.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}", @payload)
       redis.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}:local", @payload) if @status.local?
@@ -123,6 +131,8 @@ class RemoveStatusService < BaseService
   def remove_from_public
     return unless @status.public_visibility?
 
+    return if skip_streaming?
+
     redis.publish('timeline:public', @payload)
     redis.publish(@status.local? ? 'timeline:public:local' : 'timeline:public:remote', @payload)
   end
@@ -130,6 +140,8 @@ class RemoveStatusService < BaseService
   def remove_from_media
     return unless @status.public_visibility?
 
+    return if skip_streaming?
+
     redis.publish('timeline:public:media', @payload)
     redis.publish(@status.local? ? 'timeline:public:local:media' : 'timeline:public:remote:media', @payload)
   end
@@ -143,4 +155,8 @@ class RemoveStatusService < BaseService
   def permanently?
     @options[:immediate] || !(@options[:preserve] || @status.reported?)
   end
+
+  def skip_streaming?
+    !!@options[:skip_streaming]
+  end
 end

app/services/resolve_url_service.rb

@@ -89,13 +89,28 @@ class ResolveURLService < BaseService
   def process_local_url
     recognized_params = Rails.application.routes.recognize_path(@url)
 
-    return unless recognized_params[:action] == 'show'
+    case recognized_params[:controller]
+    when 'statuses'
+      return unless recognized_params[:action] == 'show'
 
-    if recognized_params[:controller] == 'statuses'
       status = Status.find_by(id: recognized_params[:id])
       check_local_status(status)
-    elsif recognized_params[:controller] == 'accounts'
+    when 'accounts'
+      return unless recognized_params[:action] == 'show'
+
       Account.find_local(recognized_params[:username])
+    when 'home'
+      return unless recognized_params[:action] == 'index' && recognized_params[:username_with_domain].present?
+
+      if recognized_params[:any]&.match?(/\A[0-9]+\Z/)
+        status = Status.find_by(id: recognized_params[:any])
+        check_local_status(status)
+      elsif recognized_params[:any].blank?
+        username, domain = recognized_params[:username_with_domain].gsub(/\A@/, '').split('@')
+        return unless username.present? && domain.present?
+
+        Account.find_remote(username, domain)
+      end
     end
   end
 

app/validators/vote_validator.rb

@@ -3,8 +3,8 @@
 class VoteValidator < ActiveModel::Validator
   def validate(vote)
     vote.errors.add(:base, I18n.t('polls.errors.expired')) if vote.poll.expired?
-
     vote.errors.add(:base, I18n.t('polls.errors.invalid_choice')) if invalid_choice?(vote)
+    vote.errors.add(:base, I18n.t('polls.errors.self_vote')) if self_vote?(vote)
 
     if vote.poll.multiple? && vote.poll.votes.where(account: vote.account, choice: vote.choice).exists?
       vote.errors.add(:base, I18n.t('polls.errors.already_voted'))
@@ -18,4 +18,8 @@ class VoteValidator < ActiveModel::Validator
   def invalid_choice?(vote)
     vote.choice.negative? || vote.choice >= vote.poll.options.size
   end
+
+  def self_vote?(vote)
+    vote.account_id == vote.poll.account_id
+  end
 end

app/views/admin/dashboard/index.html.haml

@@ -12,7 +12,7 @@
 - unless @system_checks.empty?
   .flash-message-stack
     - @system_checks.each do |message|
-      .flash-message.warning
+      .flash-message{ class: message.critical ? 'alert' : 'warning' }
         = t("admin.system_checks.#{message.key}.message_html", value: message.value ? content_tag(:strong, message.value) : nil)
         - if message.action
           = link_to t("admin.system_checks.#{message.key}.action"), message.action

app/views/admin/reports/actions/preview.html.haml

@@ -54,15 +54,15 @@
             .strike-card__statuses-list__item
               - if (status = status_map[status_id.to_i])
                 .one-liner
-                  = link_to short_account_status_url(@report.target_account, status_id), class: 'emojify' do
-                    = one_line_preview(status)
+                  .emojify= one_line_preview(status)
 
-                    - status.ordered_media_attachments.each do |media_attachment|
-                      %abbr{ title: media_attachment.description }
-                        = fa_icon 'link'
-                        = media_attachment.file_file_name
+                  - status.ordered_media_attachments.each do |media_attachment|
+                    %abbr{ title: media_attachment.description }
+                      = fa_icon 'link'
+                      = media_attachment.file_file_name
                 .strike-card__statuses-list__item__meta
-                  %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
+                  = link_to ActivityPub::TagManager.instance.url_for(status), target: '_blank' do
+                    %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
                   - unless status.application.nil?
                     ·
                     = status.application.name

app/views/admin/statuses/show.html.haml

@@ -34,7 +34,7 @@
           %td
             - if @status.trend.allowed?
               %abbr{ title: t('admin.trends.tags.current_score', score: @status.trend.score) }= t('admin.trends.tags.trending_rank', rank: @status.trend.rank)
-            - elsif @status.trend.requires_review?
+            - elsif @status.requires_review?
               = t('admin.trends.pending_review')
             - else
               = t('admin.trends.not_allowed_to_trend')

app/views/admin/webhooks/_form.html.haml

@@ -5,7 +5,7 @@
     = f.input :url, wrapper: :with_block_label, input_html: { placeholder: 'https://' }
 
   .fields-group
-    = f.input :events, collection: Webhook::EVENTS, wrapper: :with_block_label, include_blank: false, as: :check_boxes, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li'
+    = f.input :events, collection: Webhook::EVENTS, wrapper: :with_block_label, include_blank: false, as: :check_boxes, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li', disabled: Webhook::EVENTS.filter { |event| !current_user.role.can?(Webhook.permission_for_event(event)) }
 
   .actions
     = f.button :button, @webhook.new_record? ? t('admin.webhooks.add_new') : t('generic.save_changes'), type: :submit

app/views/application/_sidebar.html.haml

@@ -3,7 +3,7 @@
     = image_tag @instance_presenter.thumbnail&.file&.url(:'@1x') || asset_pack_path('media/images/preview.png'), alt: @instance_presenter.title
 
   .hero-widget__text
-    %p= @instance_presenter.description.html_safe.presence || t('about.about_mastodon_html')
+    %p= @instance_presenter.description.presence || t('about.about_mastodon_html')
 
 - if Setting.trends && !(user_signed_in? && !current_user.setting_trends)
   - trends = Trends.tags.query.allowed.limit(3)

app/views/disputes/strikes/show.html.haml

@@ -50,15 +50,15 @@
             .strike-card__statuses-list__item
               - if (status = status_map[status_id.to_i])
                 .one-liner
-                  = link_to short_account_status_url(@strike.target_account, status_id), class: 'emojify' do
-                    = one_line_preview(status)
+                  .emojify= one_line_preview(status)
 
-                    - status.ordered_media_attachments.each do |media_attachment|
-                      %abbr{ title: media_attachment.description }
-                        = fa_icon 'link'
-                        = media_attachment.file_file_name
+                  - status.ordered_media_attachments.each do |media_attachment|
+                    %abbr{ title: media_attachment.description }
+                      = fa_icon 'link'
+                      = media_attachment.file_file_name
                 .strike-card__statuses-list__item__meta
-                  %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
+                  = link_to ActivityPub::TagManager.instance.url_for(status), target: '_blank' do
+                    %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
                   - unless status.application.nil?
                     ·
                     = status.application.name

app/views/oauth/authorized_applications/index.html.haml

@@ -18,8 +18,8 @@
 
       .announcements-list__item__action-bar
         .announcements-list__item__meta
-          - if application.most_recently_used_access_token
-            = t('doorkeeper.authorized_applications.index.last_used_at', date: l(application.most_recently_used_access_token.last_used_at.to_date))
+          - if @last_used_at_by_app[application.id]
+            = t('doorkeeper.authorized_applications.index.last_used_at', date: l(@last_used_at_by_app[application.id].to_date))
           - else
             = t('doorkeeper.authorized_applications.index.never_used')
 

app/views/relationships/show.html.haml

@@ -48,7 +48,7 @@
 
         = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_followers')]), name: :remove_from_followers, class: 'table-action-link', type: :submit, data: { confirm: t('relationships.confirm_remove_selected_followers') } unless following_relationship?
 
-        = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_domains')]), name: :block_domains, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } if followed_by_relationship?
+        = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_domains')]), name: :remove_domains_from_followers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } if followed_by_relationship?
     .batch-table__body
       - if @accounts.empty?
         = nothing_here 'nothing-here--under-tabs'

app/views/settings/exports/show.html.haml

@@ -64,6 +64,6 @@
             %td= l backup.created_at
             - if backup.processed?
               %td= number_to_human_size backup.dump_file_size
-              %td= table_link_to 'download', t('exports.archive_takeout.download'), backup.dump.url
+              %td= table_link_to 'download', t('exports.archive_takeout.download'), download_backup_url(backup)
             - else
               %td{ colspan: 2 }= t('exports.archive_takeout.in_progress')

app/views/shared/_og.html.haml

@@ -1,5 +1,5 @@
 - thumbnail     = @instance_presenter.thumbnail
-- description ||= strip_tags(@instance_presenter.description.presence || t('about.about_mastodon_html'))
+- description ||= @instance_presenter.description.presence || strip_tags(t('about.about_mastodon_html'))
 
 %meta{ name: 'description', content: description }/
 

app/views/user_mailer/backup_ready.html.haml

@@ -55,5 +55,5 @@
                             %tbody
                               %tr
                                 %td.button-primary
-                                  = link_to full_asset_url(@backup.dump.url) do
+                                  = link_to download_backup_url(@backup) do
                                     %span= t 'exports.archive_takeout.download'

app/views/user_mailer/backup_ready.text.erb

@@ -4,4 +4,4 @@
 
 <%= t 'user_mailer.backup_ready.explanation' %>
 
-=> <%= full_asset_url(@backup.dump.url) %>
+=> <%= download_backup_url(@backup) %>

app/workers/activitypub/delivery_worker.rb

@@ -10,6 +10,16 @@ class ActivityPub::DeliveryWorker
 
   sidekiq_options queue: 'push', retry: 16, dead: false
 
+  # Unfortunately, we cannot control Sidekiq's jitter, so add our own
+  sidekiq_retry_in do |count|
+    # This is Sidekiq's default delay
+    delay  = (count**4) + 15
+    # Our custom jitter, that will be added to Sidekiq's built-in one.
+    # Sidekiq's built-in jitter is `rand(10) * (count + 1)`
+    jitter = rand(0.5 * (count**4))
+    delay + jitter
+  end
+
   HEADERS = { 'Content-Type' => 'application/activity+json' }.freeze
 
   def perform(json, source_account_id, inbox_url, options = {})

app/workers/activitypub/migrated_follow_delivery_worker.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class ActivityPub::MigratedFollowDeliveryWorker < ActivityPub::DeliveryWorker
+  def perform(json, source_account_id, inbox_url, old_target_account_id, options = {})
+    super(json, source_account_id, inbox_url, options)
+    unfollow_old_account!(old_target_account_id)
+  end
+
+  private
+
+  def unfollow_old_account!(old_target_account_id)
+    old_target_account = Account.find(old_target_account_id)
+    UnfollowService.new.call(@source_account, old_target_account, skip_unmerge: true)
+  rescue StandardError
+    true
+  end
+end

app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb

@@ -7,52 +7,68 @@ class Scheduler::AccountsStatusesCleanupScheduler
   # This limit is mostly to be nice to the fediverse at large and not
   # generate too much traffic.
   # This also helps limiting the running time of the scheduler itself.
-  MAX_BUDGET         = 50
+  MAX_BUDGET         = 300
 
-  # This is an attempt to spread the load across instances, as various
-  # accounts are likely to have various followers.
+  # This is an attempt to spread the load across remote servers, as
+  # spreading deletions across diverse accounts is likely to spread
+  # the deletion across diverse followers. It also helps each individual
+  # user see some effect sooner.
   PER_ACCOUNT_BUDGET = 5
 
   # This is an attempt to limit the workload generated by status removal
-  # jobs to something the particular instance can handle.
+  # jobs to something the particular server can handle.
   PER_THREAD_BUDGET  = 5
 
-  # Those avoid loading an instance that is already under load
-  MAX_DEFAULT_SIZE    = 2
-  MAX_DEFAULT_LATENCY = 5
-  MAX_PUSH_SIZE       = 5
-  MAX_PUSH_LATENCY    = 10
-  # 'pull' queue has lower priority jobs, and it's unlikely that pushing
-  # deletes would cause much issues with this queue if it didn't cause issues
-  # with default and push. Yet, do not enqueue deletes if the instance is
-  # lagging behind too much.
-  MAX_PULL_SIZE       = 500
-  MAX_PULL_LATENCY    = 300
+  # These are latency limits on various queues above which a server is
+  # considered to be under load, causing the auto-deletion to be entirely
+  # skipped for that run.
+  LOAD_LATENCY_THRESHOLDS = {
+    default: 5,
+    push: 10,
+    # The `pull` queue has lower priority jobs, and it's unlikely that
+    # pushing deletes would cause much issues with this queue if it didn't
+    # cause issues with `default` and `push`. Yet, do not enqueue deletes
+    # if the instance is lagging behind too much.
+    pull: 5.minutes.to_i,
+  }.freeze
 
-  # This is less of an issue in general, but deleting old statuses is likely
-  # to cause delivery errors, and thus increase the number of jobs to be retried.
-  # This doesn't directly translate to load, but connection errors and a high
-  # number of dead instances may lead to this spiraling out of control if
-  # unchecked.
-  MAX_RETRY_SIZE = 50_000
-
-  sidekiq_options retry: 0, lock: :until_executed
+  sidekiq_options retry: 0, lock: :until_executed, lock_ttl: 1.day.to_i
 
   def perform
     return if under_load?
 
     budget = compute_budget
-    first_policy_id = last_processed_id
+
+    # If the budget allows it, we want to consider all accounts with enabled
+    # auto cleanup at least once.
+    #
+    # We start from `first_policy_id` (the last processed id in the previous
+    # run) and process each policy until we loop to `first_policy_id`,
+    # recording into `affected_policies` any policy that caused posts to be
+    # deleted.
+    #
+    # After that, we set `full_iteration` to `false` and continue looping on
+    # policies from `affected_policies`.
+    first_policy_id   = last_processed_id || 0
+    first_iteration   = true
+    full_iteration    = true
+    affected_policies = []
 
     loop do
       num_processed_accounts = 0
 
-      scope = AccountStatusesCleanupPolicy.where(enabled: true)
-      scope.where(Account.arel_table[:id].gt(first_policy_id)) if first_policy_id.present?
+      scope = cleanup_policies(first_policy_id, affected_policies, first_iteration, full_iteration)
       scope.find_each(order: :asc) do |policy|
         num_deleted = AccountStatusesCleanupService.new.call(policy, [budget, PER_ACCOUNT_BUDGET].min)
-        num_processed_accounts += 1 unless num_deleted.zero?
         budget -= num_deleted
+
+        unless num_deleted.zero?
+          num_processed_accounts += 1
+          affected_policies << policy.id if full_iteration
+        end
+
+        full_iteration = false if !first_iteration && policy.id >= first_policy_id
+
         if budget.zero?
           save_last_processed_id(policy.id)
           break
@@ -61,37 +77,55 @@ class Scheduler::AccountsStatusesCleanupScheduler
 
       # The idea here is to loop through all policies at least once until the budget is exhausted
       # and start back after the last processed account otherwise
-      break if budget.zero? || (num_processed_accounts.zero? && first_policy_id.nil?)
-      first_policy_id = nil
+      break if budget.zero? || (num_processed_accounts.zero? && !full_iteration)
+
+      full_iteration  = false unless first_iteration
+      first_iteration = false
     end
   end
 
   def compute_budget
-    threads = Sidekiq::ProcessSet.new.select { |x| x['queues'].include?('push') }.map { |x| x['concurrency'] }.sum
+    # Each post deletion is a `RemovalWorker` job (on `default` queue), each
+    # potentially spawning many `ActivityPub::DeliveryWorker` jobs (on the `push` queue).
+    threads = Sidekiq::ProcessSet.new.select { |x| x['queues'].include?('push') }.pluck('concurrency').sum
     [PER_THREAD_BUDGET * threads, MAX_BUDGET].min
   end
 
   def under_load?
-    return true if Sidekiq::Stats.new.retry_size > MAX_RETRY_SIZE
-    queue_under_load?('default', MAX_DEFAULT_SIZE, MAX_DEFAULT_LATENCY) || queue_under_load?('push', MAX_PUSH_SIZE, MAX_PUSH_LATENCY) || queue_under_load?('pull', MAX_PULL_SIZE, MAX_PULL_LATENCY)
+    LOAD_LATENCY_THRESHOLDS.any? { |queue, max_latency| queue_under_load?(queue, max_latency) }
   end
 
   private
 
-  def queue_under_load?(name, max_size, max_latency)
-    queue = Sidekiq::Queue.new(name)
-    queue.size > max_size || queue.latency > max_latency
+  def cleanup_policies(first_policy_id, affected_policies, first_iteration, full_iteration)
+    scope = AccountStatusesCleanupPolicy.where(enabled: true)
+
+    if full_iteration
+      # If we are doing a full iteration, examine all policies we have not examined yet
+      if first_iteration
+        scope.where(id: first_policy_id...)
+      else
+        scope.where(id: ..first_policy_id).or(scope.where(id: affected_policies))
+      end
+    else
+      # Otherwise, examine only policies that previously yielded posts to delete
+      scope.where(id: affected_policies)
+    end
+  end
+
+  def queue_under_load?(name, max_latency)
+    Sidekiq::Queue.new(name).latency > max_latency
   end
 
   def last_processed_id
-    redis.get('account_statuses_cleanup_scheduler:last_account_id')
+    redis.get('account_statuses_cleanup_scheduler:last_policy_id')&.to_i
   end
 
   def save_last_processed_id(id)
     if id.nil?
-      redis.del('account_statuses_cleanup_scheduler:last_account_id')
+      redis.del('account_statuses_cleanup_scheduler:last_policy_id')
     else
-      redis.set('account_statuses_cleanup_scheduler:last_account_id', id, ex: 1.hour.seconds)
+      redis.set('account_statuses_cleanup_scheduler:last_policy_id', id, ex: 1.hour.seconds)
     end
   end
 end

app/workers/scheduler/indexing_scheduler.rb

@@ -6,17 +6,19 @@ class Scheduler::IndexingScheduler
 
   sidekiq_options retry: 0
 
+  IMPORT_BATCH_SIZE = 1000
+  SCAN_BATCH_SIZE = 10 * IMPORT_BATCH_SIZE
+
   def perform
     return unless Chewy.enabled?
 
     indexes.each do |type|
       with_redis do |redis|
-        ids = redis.smembers("chewy:queue:#{type.name}")
-
-        type.import!(ids)
-
-        redis.pipelined do |pipeline|
-          ids.each { |id| pipeline.srem("chewy:queue:#{type.name}", id) }
+        redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids|
+          type.import!(ids)
+          redis.pipelined do |pipeline|
+            pipeline.srem("chewy:queue:#{type.name}", ids)
+          end
         end
       end
     end

app/workers/scheduler/user_cleanup_scheduler.rb

@@ -24,7 +24,7 @@ class Scheduler::UserCleanupScheduler
   def clean_discarded_statuses!
     Status.unscoped.discarded.where('deleted_at <= ?', 30.days.ago).find_in_batches do |statuses|
       RemovalWorker.push_bulk(statuses) do |status|
-        [status.id, { 'immediate' => true }]
+        [status.id, { 'immediate' => true, 'skip_streaming' => true }]
       end
     end
   end

app/workers/unfollow_follow_worker.rb

@@ -10,13 +10,7 @@ class UnfollowFollowWorker
     old_target_account = Account.find(old_target_account_id)
     new_target_account = Account.find(new_target_account_id)
 
-    follow    = follower_account.active_relationships.find_by(target_account: old_target_account)
-    reblogs   = follow&.show_reblogs?
-    notify    = follow&.notify?
-    languages = follow&.languages
-
-    FollowService.new.call(follower_account, new_target_account, reblogs: reblogs, notify: notify, languages: languages, bypass_locked: bypass_locked, bypass_limit: true)
-    UnfollowService.new.call(follower_account, old_target_account, skip_unmerge: true)
+    FollowMigrationService.new.call(follower_account, new_target_account, old_target_account, bypass_locked: bypass_locked)
   rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError
     true
   end

bin/tootctl

@@ -5,7 +5,9 @@ require_relative '../config/boot'
 require_relative '../lib/cli'
 
 begin
-  Mastodon::CLI.start(ARGV)
+  Chewy.strategy(:mastodon) do
+    Mastodon::CLI.start(ARGV)
+  end
 rescue Interrupt
   exit(130)
 end

config/application.rb

@@ -28,6 +28,7 @@ require_relative '../lib/paperclip/url_generator_extensions'
 require_relative '../lib/paperclip/attachment_extensions'
 require_relative '../lib/paperclip/lazy_thumbnail'
 require_relative '../lib/paperclip/gif_transcoder'
+require_relative '../lib/paperclip/media_type_spoof_detector_extensions'
 require_relative '../lib/paperclip/transcoder'
 require_relative '../lib/paperclip/type_corrector'
 require_relative '../lib/paperclip/response_with_limit_adapter'
@@ -35,9 +36,11 @@ require_relative '../lib/terrapin/multi_pipe_extensions'
 require_relative '../lib/mastodon/snowflake'
 require_relative '../lib/mastodon/version'
 require_relative '../lib/mastodon/rack_middleware'
+require_relative '../lib/public_file_server_middleware'
 require_relative '../lib/devise/two_factor_ldap_authenticatable'
 require_relative '../lib/devise/two_factor_pam_authenticatable'
 require_relative '../lib/chewy/strategy/mastodon'
+require_relative '../lib/chewy/strategy/bypass_with_warning'
 require_relative '../lib/webpacker/manifest_extensions'
 require_relative '../lib/webpacker/helper_extensions'
 require_relative '../lib/rails/engine_extensions'
@@ -181,6 +184,10 @@ module Mastodon
     config.active_job.queue_adapter = :sidekiq
     config.action_mailer.deliver_later_queue_name = 'mailers'
 
+    # We use our own middleware for this
+    config.public_file_server.enabled = false
+
+    config.middleware.use PublicFileServerMiddleware if Rails.env.development? || ENV['RAILS_SERVE_STATIC_FILES'] == 'true'
     config.middleware.use Rack::Attack
     config.middleware.use Mastodon::RackMiddleware
 

config/database.yml

@@ -5,6 +5,7 @@ default: &default
   connect_timeout: 15
   encoding: unicode
   sslmode: <%= ENV['DB_SSLMODE'] || "prefer" %>
+  application_name: ''
 
 development:
   <<: *default

config/environments/development.rb

@@ -16,12 +16,7 @@ Rails.application.configure do
   # Run rails dev:cache to toggle caching.
   if Rails.root.join('tmp/caching-dev.txt').exist?
     config.action_controller.perform_caching = true
-
     config.cache_store = :redis_cache_store, REDIS_CACHE_PARAMS
-
-    config.public_file_server.headers = {
-      'Cache-Control' => "public, max-age=#{2.days.to_i}",
-    }
   else
     config.action_controller.perform_caching = false
 

config/environments/production.rb

@@ -19,27 +19,16 @@ Rails.application.configure do
   # or in config/master.key. This key is used to decrypt credentials (and other encrypted files).
   # config.require_master_key = true
 
-  # Disable serving static files from the `/public` folder by default since
-  # Apache or NGINX already handles this.
-  config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present?
-
   ActiveSupport::Logger.new(STDOUT).tap do |logger|
     logger.formatter = config.log_formatter
     config.logger = ActiveSupport::TaggedLogging.new(logger)
   end
 
-  # Compress JavaScripts and CSS.
-  # config.assets.js_compressor = Uglifier.new(mangle: false)
-  # config.assets.css_compressor = :sass
-
   # Do not fallback to assets pipeline if a precompiled asset is missed.
   config.assets.compile = false
 
-  # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb
-
   # Specifies the header that your server uses for sending files.
-  # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache
-  config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX
+  config.action_dispatch.x_sendfile_header = ENV['SENDFILE_HEADER'] if ENV['SENDFILE_HEADER'].present?
 
   # Allow to specify public IP of reverse proxy if it's needed
   config.action_dispatch.trusted_proxies = ENV['TRUSTED_PROXY_IP'].split(/(?:\s*,\s*|\s+)/).map { |item| IPAddr.new(item) } if ENV['TRUSTED_PROXY_IP'].present?
@@ -67,7 +56,7 @@ Rails.application.configure do
 
   # Enable locale fallbacks for I18n (makes lookups for any locale fall back to
   # English when a translation cannot be found).
-  config.i18n.fallbacks = [:en]
+  config.i18n.fallbacks = true
 
   # Send deprecation notices to registered listeners.
   config.active_support.deprecation = :notify
@@ -128,6 +117,7 @@ Rails.application.configure do
     enable_starttls_auto: enable_starttls_auto,
     tls: ENV['SMTP_TLS'].presence && ENV['SMTP_TLS'] == 'true',
     ssl: ENV['SMTP_SSL'].presence && ENV['SMTP_SSL'] == 'true',
+    read_timeout: 20,
   }
 
   config.action_mailer.delivery_method = ENV.fetch('SMTP_DELIVERY_METHOD', 'smtp').to_sym

config/environments/test.rb

@@ -12,11 +12,6 @@ Rails.application.configure do
   # preloads Rails for running tests, you may have to set it to true.
   config.eager_load = false
 
-  # Configure public file server for tests with Cache-Control for performance.
-  config.public_file_server.enabled = true
-  config.public_file_server.headers = {
-    'Cache-Control' => "public, max-age=#{1.hour.to_i}"
-  }
   config.assets.digest = false
 
   # Show full error reports and disable caching.

config/imagemagick/policy.xml

@@ -0,0 +1,27 @@
+<policymap>
+  <!-- Set some basic system resource limits -->
+  <policy domain="resource" name="time" value="60" />
+
+  <policy domain="module" rights="none" pattern="URL" />
+
+  <policy domain="filter" rights="none" pattern="*" />
+
+  <!--
+    Ideally, we would restrict ImageMagick to only accessing its own
+    disk-backed pixel cache as well as Mastodon-created Tempfiles.
+
+    However, those paths depend on the operating system and environment
+    variables, so they can only be known at runtime.
+
+    Furthermore, those paths are not necessarily shared across Mastodon
+    processes, so even creating a policy.xml at runtime is impractical.
+
+    For the time being, only disable indirect reads.
+  -->
+  <policy domain="path" rights="none" pattern="@*" />
+
+  <!-- Disallow any coder by default, and only enable ones required by Mastodon -->
+  <policy domain="coder" rights="none" pattern="*" />
+  <policy domain="coder" rights="read | write" pattern="{PNG,JPEG,GIF,HEIC,WEBP}" />
+  <policy domain="coder" rights="write" pattern="{HISTOGRAM,RGB,INFO}" />
+</policymap>

config/initializers/chewy.rb

@@ -19,7 +19,7 @@ Chewy.settings = {
 # cycle, which takes care of checking if Elasticsearch is enabled
 # or not. However, mind that for the Rails console, the :urgent
 # strategy is set automatically with no way to override it.
-Chewy.root_strategy              = :mastodon
+Chewy.root_strategy              = :bypass_with_warning if Rails.env.production?
 Chewy.request_strategy           = :mastodon
 Chewy.use_after_commit_callbacks = false
 

config/initializers/content_security_policy.rb

@@ -3,7 +3,7 @@
 # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy
 
 def host_to_url(str)
-  "http#{Rails.configuration.x.use_https ? 's' : ''}://#{str}" unless str.blank?
+  "http#{Rails.configuration.x.use_https ? 's' : ''}://#{str}".split('/').first if str.present?
 end
 
 base_host = Rails.configuration.x.web_domain

config/initializers/paperclip.rb

@@ -124,6 +124,7 @@ elsif ENV['SWIFT_ENABLED'] == 'true'
       openstack_domain_name: ENV.fetch('SWIFT_DOMAIN_NAME') { 'default' },
       openstack_region: ENV['SWIFT_REGION'],
       openstack_cache_ttl: ENV.fetch('SWIFT_CACHE_TTL') { 60 },
+      openstack_temp_url_key: ENV['SWIFT_TEMP_URL_KEY'],
     },
     
     fog_file: { 'Cache-Control' => 'public, max-age=315576000, immutable' },
@@ -154,3 +155,10 @@ unless defined?(Seahorse)
     end
   end
 end
+
+# Set our ImageMagick security policy, but allow admins to override it
+ENV['MAGICK_CONFIGURE_PATH'] = begin
+  imagemagick_config_paths = ENV.fetch('MAGICK_CONFIGURE_PATH', '').split(File::PATH_SEPARATOR)
+  imagemagick_config_paths << Rails.root.join('config', 'imagemagick').expand_path.to_s
+  imagemagick_config_paths.join(File::PATH_SEPARATOR)
+end

config/initializers/twitter_regex.rb

@@ -25,7 +25,7 @@ module Twitter::TwitterText
       \)
     /iox
     UCHARS = '\u{A0}-\u{D7FF}\u{F900}-\u{FDCF}\u{FDF0}-\u{FFEF}\u{10000}-\u{1FFFD}\u{20000}-\u{2FFFD}\u{30000}-\u{3FFFD}\u{40000}-\u{4FFFD}\u{50000}-\u{5FFFD}\u{60000}-\u{6FFFD}\u{70000}-\u{7FFFD}\u{80000}-\u{8FFFD}\u{90000}-\u{9FFFD}\u{A0000}-\u{AFFFD}\u{B0000}-\u{BFFFD}\u{C0000}-\u{CFFFD}\u{D0000}-\u{DFFFD}\u{E1000}-\u{EFFFD}\u{E000}-\u{F8FF}\u{F0000}-\u{FFFFD}\u{100000}-\u{10FFFD}'
-    REGEXEN[:valid_url_query_chars] = /[a-z0-9!?\*'\(\);:&=\+\$\/%#\[\]\-_\.,~|@#{UCHARS}]/iou
+    REGEXEN[:valid_url_query_chars] = /[a-z0-9!?\*'\(\);:&=\+\$\/%#\[\]\-_\.,~|@\^#{UCHARS}]/iou
     REGEXEN[:valid_url_query_ending_chars] = /[a-z0-9_&=#\/\-#{UCHARS}]/iou
     REGEXEN[:valid_url_path] = /(?:
       (?:

config/locales/activerecord.en.yml

@@ -53,3 +53,7 @@ en:
             position:
               elevated: cannot be higher than your current role
               own_role: cannot be changed with your current role
+        webhook:
+          attributes:
+            events:
+              invalid_permissions: cannot include events you don't have the rights to

config/locales/en.yml

@@ -805,6 +805,12 @@ en:
         message_html: You haven't defined any server rules.
       sidekiq_process_check:
         message_html: No Sidekiq process running for the %{value} queue(s). Please review your Sidekiq configuration
+      upload_check_privacy_error:
+        action: Check here for more information
+        message_html: "<strong>Your web server is misconfigured. The privacy of your users is at risk.</strong>"
+      upload_check_privacy_error_object_storage:
+        action: Check here for more information
+        message_html: "<strong>Your object storage is misconfigured. The privacy of your users is at risk.</strong>"
     tags:
       review: Review status
       updated_msg: Hashtag settings updated successfully
@@ -1381,6 +1387,7 @@ en:
       expired: The poll has already ended
       invalid_choice: The chosen vote option does not exist
       over_character_limit: cannot be longer than %{max} characters each
+      self_vote: You cannot vote in your own polls
       too_few_options: must have more than one item
       too_many_options: can't contain more than %{max} items
   preferences:
@@ -1399,6 +1406,7 @@ en:
     confirm_remove_selected_followers: Are you sure you want to remove selected followers?
     confirm_remove_selected_follows: Are you sure you want to remove selected follows?
     dormant: Dormant
+    follow_failure: Could not follow some of the selected accounts.
     follow_selected_followers: Follow selected followers
     followers: Followers
     following: Following

config/puma.rb

@@ -22,3 +22,5 @@ on_worker_boot do
 end
 
 plugin :tmp_restart
+
+set_remote_address(proxy_protocol: :v1) if ENV['PROXY_PROTO_V1'] == 'true'

config/routes.rb

@@ -110,6 +110,8 @@ Rails.application.routes.draw do
 
   resource :inbox, only: [:create], module: :activitypub
 
+  get '/:encoded_at(*path)', to: redirect("/@%{path}"), constraints: { encoded_at: /%40/ }
+
   constraints(username: /[^@\/.]+/) do
     get '/@:username', to: 'accounts#show', as: :short_account
     get '/@:username/with_replies', to: 'accounts#show', as: :short_account_with_replies
@@ -218,6 +220,7 @@ Rails.application.routes.draw do
   resource :statuses_cleanup, controller: :statuses_cleanup, only: [:show, :update]
 
   get '/media_proxy/:id/(*any)', to: 'media_proxy#show', as: :media_proxy, format: false
+  get '/backups/:id/download', to: 'backups#download', as: :download_backup, format: false
 
   resource :authorize_interaction, only: [:show, :create]
   resource :share, only: [:show, :create]
@@ -470,7 +473,9 @@ Rails.application.routes.draw do
         resources :list, only: :show
       end
 
-      resources :streaming, only: [:index]
+      get '/streaming', to: 'streaming#index'
+      get '/streaming/(*any)', to: 'streaming#index'
+
       resources :custom_emojis, only: [:index]
       resources :suggestions, only: [:index, :destroy]
       resources :scheduled_statuses, only: [:index, :show, :update, :destroy]

db/seeds.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
-Dir[Rails.root.join('db', 'seeds', '*.rb')].sort.each do |seed|
-  load seed
+Chewy.strategy(:mastodon) do
+  Dir[Rails.root.join('db', 'seeds', '*.rb')].sort.each do |seed|
+    load seed
+  end
 end

dist/nginx.conf

@@ -109,6 +109,8 @@ server {
   location ~ ^/system/ {
     add_header Cache-Control "public, max-age=2419200, immutable";
     add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
+    add_header X-Content-Type-Options nosniff;
+    add_header Content-Security-Policy "default-src 'none'; form-action 'none'";
     try_files $uri =404;
   }
 

docker-compose.yml

@@ -56,7 +56,7 @@ services:
 
   web:
     build: .
-    image: tootsuite/mastodon
+    image: ghcr.io/mastodon/mastodon
     restart: always
     env_file: .env.production
     command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@@ -77,7 +77,7 @@ services:
 
   streaming:
     build: .
-    image: tootsuite/mastodon
+    image: ghcr.io/mastodon/mastodon
     restart: always
     env_file: .env.production
     command: node ./streaming
@@ -95,7 +95,7 @@ services:
 
   sidekiq:
     build: .
-    image: tootsuite/mastodon
+    image: ghcr.io/mastodon/mastodon
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq

lib/chewy/strategy/bypass_with_warning.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Chewy
+  class Strategy
+    class BypassWithWarning < Base
+      def update(...)
+        Rails.logger.warn 'Chewy update without a root strategy' unless @warning_issued
+        @warning_issued = true
+      end
+    end
+  end
+end

lib/mastodon/accounts_cli.rb

@@ -372,16 +372,16 @@ module Mastodon
     option :concurrency, type: :numeric, default: 5, aliases: [:c]
     option :verbose, type: :boolean, aliases: [:v]
     option :dry_run, type: :boolean
-    desc 'refresh [USERNAME]', 'Fetch remote user data and files'
+    desc 'refresh [USERNAMES]', 'Fetch remote user data and files'
     long_desc <<-LONG_DESC
       Fetch remote user data and files for one or multiple accounts.
 
       With the --all option, all remote accounts will be processed.
       Through the --domain option, this can be narrowed down to a
-      specific domain only. Otherwise, a single remote account must
-      be specified with USERNAME.
+      specific domain only. Otherwise, remote accounts must be
+      specified with space-separated USERNAMES.
     LONG_DESC
-    def refresh(username = nil)
+    def refresh(*usernames)
       dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
 
       if options[:domain] || options[:all]
@@ -397,19 +397,25 @@ module Mastodon
         end
 
         say("Refreshed #{processed} accounts#{dry_run}", :green, true)
-      elsif username.present?
-        username, domain = username.split('@')
-        account = Account.find_remote(username, domain)
+      elsif !usernames.empty?
+        usernames.each do |user|
+          user, domain = user.split('@')
+          account = Account.find_remote(user, domain)
 
-        if account.nil?
-          say('No such account', :red)
-          exit(1)
-        end
+          if account.nil?
+            say('No such account', :red)
+            exit(1)
+          end
 
-        unless options[:dry_run]
-          account.reset_avatar!
-          account.reset_header!
-          account.save
+          next if options[:dry_run]
+
+          begin
+            account.reset_avatar!
+            account.reset_header!
+            account.save
+          rescue Mastodon::UnexpectedResponseError
+            say("Account failed: #{user}@#{domain}", :red)
+          end
         end
 
         say("OK#{dry_run}", :green)
@@ -536,7 +542,7 @@ module Mastodon
         User.pending.find_each(&:approve!)
         say('OK', :green)
       elsif options[:number]
-        User.pending.limit(options[:number]).each(&:approve!)
+        User.pending.order(created_at: :asc).limit(options[:number]).each(&:approve!)
         say('OK', :green)
       elsif username.present?
         account = Account.find_local(username)
@@ -631,7 +637,7 @@ module Mastodon
           exit(1)
         end
 
-        unless options[:force] || migration.target_acount_id == account.moved_to_account_id
+        unless options[:force] || migration.target_account_id == account.moved_to_account_id
           say('The specified account is not redirecting to its last migration target. Use --force if you want to replay the migration anyway', :red)
           exit(1)
         end