From 520b2086af066d6434a6f2652870a6bb620f4e23 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 12 Oct 2023 16:47:18 +0200 Subject: [PATCH 01/17] Change PWA start URL from `/home` to `/` (#27377) --- app/serializers/manifest_serializer.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/serializers/manifest_serializer.rb b/app/serializers/manifest_serializer.rb index 48f3aa7a6a..b6533815a5 100644 --- a/app/serializers/manifest_serializer.rb +++ b/app/serializers/manifest_serializer.rb @@ -16,11 +16,18 @@ class ManifestSerializer < ActiveModel::Serializer 512 ).freeze - attributes :name, :short_name, + attributes :id, :name, :short_name, :icons, :theme_color, :background_color, :display, :start_url, :scope, :share_target, :shortcuts + def id + # This is set to `/home` because that was the old value of `start_url` and + # thus the fallback ID computed by Chrome: + # https://developer.chrome.com/blog/pwa-manifest-id/ + '/home' + end + def name object.title end @@ -53,7 +60,7 @@ class ManifestSerializer < ActiveModel::Serializer end def start_url - '/home' + '/' end def scope From 19ed22dc582e9fd922b221034953c428dd0de856 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 12 Jun 2024 09:28:28 +0200 Subject: [PATCH 02/17] Fix duplicate `@context` attribute in user export (#30653) --- app/services/backup_service.rb | 4 ++-- spec/services/backup_service_spec.rb | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/app/services/backup_service.rb b/app/services/backup_service.rb index 3ef0366c3c..b3282c409b 100644 --- a/app/services/backup_service.rb +++ b/app/services/backup_service.rb @@ -19,8 +19,8 @@ class BackupService < BaseService def build_outbox_json!(file) skeleton = serialize(collection_presenter, ActivityPub::CollectionSerializer) - skeleton[:@context] = full_context - skeleton[:orderedItems] = ['!PLACEHOLDER!'] + skeleton['@context'] = full_context + skeleton['orderedItems'] = ['!PLACEHOLDER!'] skeleton = Oj.dump(skeleton) prepend, append = skeleton.split('"!PLACEHOLDER!"') add_comma = false diff --git a/spec/services/backup_service_spec.rb b/spec/services/backup_service_spec.rb index 806ba18323..1eb789d1f5 100644 --- a/spec/services/backup_service_spec.rb +++ b/spec/services/backup_service_spec.rb @@ -55,9 +55,11 @@ RSpec.describe BackupService, type: :service do end def expect_outbox_export - json = export_json(:outbox) + body = export_json_raw(:outbox) + json = Oj.load(body) aggregate_failures do + expect(body.scan('@context').count).to eq 1 expect(json['@context']).to_not be_nil expect(json['type']).to eq 'OrderedCollection' expect(json['totalItems']).to eq 2 @@ -85,8 +87,12 @@ RSpec.describe BackupService, type: :service do end end + def export_json_raw(type) + read_zip_file(backup, "#{type}.json") + end + def export_json(type) - Oj.load(read_zip_file(backup, "#{type}.json")) + Oj.load(export_json_raw(type)) end def include_create_item(status) From bfc287fd6b83d9b32d0d7c34ed5d61f63c5bae35 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 20 Jun 2024 08:56:12 +0200 Subject: [PATCH 03/17] Remove dependency on posix-spawn (#18559) --- Gemfile | 1 - Gemfile.lock | 2 -- 2 files changed, 3 deletions(-) diff --git a/Gemfile b/Gemfile index 1d038ad9f8..7f7da8c57f 100644 --- a/Gemfile +++ b/Gemfile @@ -65,7 +65,6 @@ gem 'nsa' gem 'oj', '~> 3.14' gem 'ox', '~> 2.14' gem 'parslet' -gem 'posix-spawn' gem 'public_suffix', '~> 5.0' gem 'pundit', '~> 2.3' gem 'premailer-rails' diff --git a/Gemfile.lock b/Gemfile.lock index 891566e1ae..2ef61e145e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -517,7 +517,6 @@ GEM pg (1.5.5) pghero (3.3.4) activerecord (>= 6) - posix-spawn (0.3.15) premailer (1.21.0) addressable css_parser (>= 1.12.0) @@ -892,7 +891,6 @@ DEPENDENCIES parslet pg (~> 1.5) pghero - posix-spawn premailer-rails private_address_check (~> 0.5) public_suffix (~> 5.0) From 3f75c6f048c75e5edd5749c0dfe4df1b935d0d3a Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 20 Jun 2024 08:59:41 +0200 Subject: [PATCH 04/17] Update dependency rails --- Gemfile.lock | 122 +++++++++++++++++++++++++-------------------------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 2ef61e145e..b3e19cd8fd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -28,47 +28,47 @@ GIT GEM remote: https://rubygems.org/ specs: - actioncable (7.0.8.1) - actionpack (= 7.0.8.1) - activesupport (= 7.0.8.1) + actioncable (7.0.8.4) + actionpack (= 7.0.8.4) + activesupport (= 7.0.8.4) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (7.0.8.1) - actionpack (= 7.0.8.1) - activejob (= 7.0.8.1) - activerecord (= 7.0.8.1) - activestorage (= 7.0.8.1) - activesupport (= 7.0.8.1) + actionmailbox (7.0.8.4) + actionpack (= 7.0.8.4) + activejob (= 7.0.8.4) + activerecord (= 7.0.8.4) + activestorage (= 7.0.8.4) + activesupport (= 7.0.8.4) mail (>= 2.7.1) net-imap net-pop net-smtp - actionmailer (7.0.8.1) - actionpack (= 7.0.8.1) - actionview (= 7.0.8.1) - activejob (= 7.0.8.1) - activesupport (= 7.0.8.1) + actionmailer (7.0.8.4) + actionpack (= 7.0.8.4) + actionview (= 7.0.8.4) + activejob (= 7.0.8.4) + activesupport (= 7.0.8.4) mail (~> 2.5, >= 2.5.4) net-imap net-pop net-smtp rails-dom-testing (~> 2.0) - actionpack (7.0.8.1) - actionview (= 7.0.8.1) - activesupport (= 7.0.8.1) + actionpack (7.0.8.4) + actionview (= 7.0.8.4) + activesupport (= 7.0.8.4) rack (~> 2.0, >= 2.2.4) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (7.0.8.1) - actionpack (= 7.0.8.1) - activerecord (= 7.0.8.1) - activestorage (= 7.0.8.1) - activesupport (= 7.0.8.1) + actiontext (7.0.8.4) + actionpack (= 7.0.8.4) + activerecord (= 7.0.8.4) + activestorage (= 7.0.8.4) + activesupport (= 7.0.8.4) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.0.8.1) - activesupport (= 7.0.8.1) + actionview (7.0.8.4) + activesupport (= 7.0.8.4) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) @@ -78,22 +78,22 @@ GEM activemodel (>= 4.1, < 7.1) case_transform (>= 0.2) jsonapi-renderer (>= 0.1.1.beta1, < 0.3) - activejob (7.0.8.1) - activesupport (= 7.0.8.1) + activejob (7.0.8.4) + activesupport (= 7.0.8.4) globalid (>= 0.3.6) - activemodel (7.0.8.1) - activesupport (= 7.0.8.1) - activerecord (7.0.8.1) - activemodel (= 7.0.8.1) - activesupport (= 7.0.8.1) - activestorage (7.0.8.1) - actionpack (= 7.0.8.1) - activejob (= 7.0.8.1) - activerecord (= 7.0.8.1) - activesupport (= 7.0.8.1) + activemodel (7.0.8.4) + activesupport (= 7.0.8.4) + activerecord (7.0.8.4) + activemodel (= 7.0.8.4) + activesupport (= 7.0.8.4) + activestorage (7.0.8.4) + actionpack (= 7.0.8.4) + activejob (= 7.0.8.4) + activerecord (= 7.0.8.4) + activesupport (= 7.0.8.4) marcel (~> 1.0) mini_mime (>= 1.1.0) - activesupport (7.0.8.1) + activesupport (7.0.8.4) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -350,7 +350,7 @@ GEM httplog (1.6.2) rack (>= 2.0) rainbow (>= 2.0.0) - i18n (1.14.1) + i18n (1.14.5) concurrent-ruby (~> 1.0) i18n-tasks (1.0.12) activesupport (>= 4.0.2) @@ -432,7 +432,7 @@ GEM net-imap net-pop net-smtp - marcel (1.0.2) + marcel (1.0.4) mario-redis-lock (1.2.1) redis (>= 3.0.5) matrix (0.4.2) @@ -446,7 +446,7 @@ GEM mime-types-data (~> 3.2015) mime-types-data (3.2023.0808) mini_mime (1.1.5) - mini_portile2 (2.8.5) + mini_portile2 (2.8.7) minitest (5.19.0) msgpack (1.7.1) multi_json (1.15.0) @@ -468,8 +468,8 @@ GEM net-smtp (0.3.4) net-protocol net-ssh (7.1.0) - nio4r (2.7.0) - nokogiri (1.16.5) + nio4r (2.7.3) + nokogiri (1.16.6) mini_portile2 (~> 2.8.2) racc (~> 1.4) nsa (0.3.0) @@ -533,7 +533,7 @@ GEM activesupport (>= 3.0.0) raabro (1.4.0) racc (1.7.3) - rack (2.2.8.1) + rack (2.2.9) rack-attack (6.7.0) rack (>= 1.0, < 4) rack-cors (2.0.2) @@ -550,20 +550,20 @@ GEM rack rack-test (2.1.0) rack (>= 1.3) - rails (7.0.8.1) - actioncable (= 7.0.8.1) - actionmailbox (= 7.0.8.1) - actionmailer (= 7.0.8.1) - actionpack (= 7.0.8.1) - actiontext (= 7.0.8.1) - actionview (= 7.0.8.1) - activejob (= 7.0.8.1) - activemodel (= 7.0.8.1) - activerecord (= 7.0.8.1) - activestorage (= 7.0.8.1) - activesupport (= 7.0.8.1) + rails (7.0.8.4) + actioncable (= 7.0.8.4) + actionmailbox (= 7.0.8.4) + actionmailer (= 7.0.8.4) + actionpack (= 7.0.8.4) + actiontext (= 7.0.8.4) + actionview (= 7.0.8.4) + activejob (= 7.0.8.4) + activemodel (= 7.0.8.4) + activerecord (= 7.0.8.4) + activestorage (= 7.0.8.4) + activesupport (= 7.0.8.4) bundler (>= 1.15.0) - railties (= 7.0.8.1) + railties (= 7.0.8.4) rails-controller-testing (1.0.5) actionpack (>= 5.0.1.rc1) actionview (>= 5.0.1.rc1) @@ -578,9 +578,9 @@ GEM rails-i18n (7.0.7) i18n (>= 0.7, < 2) railties (>= 6.0.0, < 8) - railties (7.0.8.1) - actionpack (= 7.0.8.1) - activesupport (= 7.0.8.1) + railties (7.0.8.4) + actionpack (= 7.0.8.4) + activesupport (= 7.0.8.4) method_source rake (>= 12.2) thor (~> 1.0) @@ -743,7 +743,7 @@ GEM terrapin (0.6.0) climate_control (>= 0.0.3, < 1.0) test-prof (1.2.3) - thor (1.3.0) + thor (1.3.1) tilt (2.2.0) timeout (0.4.1) tpm-key_attestation (0.12.0) @@ -809,7 +809,7 @@ GEM xorcist (1.1.3) xpath (3.2.0) nokogiri (~> 1.8) - zeitwerk (2.6.13) + zeitwerk (2.6.16) PLATFORMS ruby From 55408f80853b69fb08d3ae8f9a7daeab78b21a54 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 20 Jun 2024 09:00:08 +0200 Subject: [PATCH 05/17] Update dependency cbor --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index b3e19cd8fd..edddcbd737 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -191,7 +191,7 @@ GEM xpath (~> 3.2) case_transform (0.2) activesupport - cbor (0.5.9.6) + cbor (0.5.9.8) charlock_holmes (0.7.7) chewy (7.3.4) activesupport (>= 5.2) From fcae9435ec6cb22e536c7b4eef351ce0b402e3ef Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 24 Jun 2024 15:11:10 +0200 Subject: [PATCH 06/17] Fix `/admin/accounts/:account_id/statuses/:id` for edited posts with media attachments (#30819) --- app/models/status_edit.rb | 2 +- spec/controllers/admin/statuses_controller_spec.rb | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/models/status_edit.rb b/app/models/status_edit.rb index 2b3248bb2c..7de509dda1 100644 --- a/app/models/status_edit.rb +++ b/app/models/status_edit.rb @@ -42,7 +42,7 @@ class StatusEdit < ApplicationRecord default_scope { order(id: :asc) } delegate :local?, :application, :edited?, :edited_at, - :discarded?, :visibility, to: :status + :discarded?, :visibility, :language, to: :status def emojis return @emojis if defined?(@emojis) diff --git a/spec/controllers/admin/statuses_controller_spec.rb b/spec/controllers/admin/statuses_controller_spec.rb index 9befdf978f..5c8fc0a639 100644 --- a/spec/controllers/admin/statuses_controller_spec.rb +++ b/spec/controllers/admin/statuses_controller_spec.rb @@ -43,6 +43,11 @@ describe Admin::StatusesController do describe 'GET #show' do before do + status.media_attachments << Fabricate(:media_attachment, type: :image, account: status.account) + status.save! + status.snapshot!(at_time: status.created_at, rate_limit: false) + status.update!(text: 'Hello, this is an edited post') + status.snapshot!(rate_limit: false) get :show, params: { account_id: account.id, id: status.id } end From 5fd7cd79e0734d0f8d7d1250d4c1629def1a1b48 Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 24 Jun 2024 15:49:39 +0200 Subject: [PATCH 07/17] Specify yarn version to avoid confusion with main which uses Yarn 4 --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 227399e804..35a236c8cc 100644 --- a/package.json +++ b/package.json @@ -226,5 +226,6 @@ "Capfile|Gemfile|*.{rb,ruby,ru,rake}": "bundle exec rubocop --force-exclusion -a", "*.{js,jsx,ts,tsx}": "eslint --fix", "*.{css,scss}": "stylelint --fix" - } + }, + "packageManager": "yarn@1.22.22" } From 1e87634a43b19953f3e2956003d176967f2e7a8d Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 28 Jun 2024 23:23:19 +0200 Subject: [PATCH 08/17] Update dependency charlock_holmes --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index edddcbd737..34958703a9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -192,7 +192,7 @@ GEM case_transform (0.2) activesupport cbor (0.5.9.8) - charlock_holmes (0.7.7) + charlock_holmes (0.7.8) chewy (7.3.4) activesupport (>= 5.2) elasticsearch (>= 7.12.0, < 7.14.0) From 17f69c0002e68851293b158e1e7cda685fdfb2dc Mon Sep 17 00:00:00 2001 From: Tim Rogers Date: Mon, 24 Jun 2024 09:41:04 -0500 Subject: [PATCH 09/17] Added check for STATSD_ADDR setting to emit a warning and proceed rather than crashing if the address is unreachable (#30691) --- config/initializers/statsd.rb | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/config/initializers/statsd.rb b/config/initializers/statsd.rb index a655c10716..f1628a9d12 100644 --- a/config/initializers/statsd.rb +++ b/config/initializers/statsd.rb @@ -3,13 +3,17 @@ if ENV['STATSD_ADDR'].present? host, port = ENV['STATSD_ADDR'].split(':') - statsd = Statsd.new(host, port) - statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') } + begin + statsd = Statsd.new(host, port) + statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') } - NSA.inform_statsd(statsd) do |informant| - informant.collect(:action_controller, :web) - informant.collect(:active_record, :db) - informant.collect(:active_support_cache, :cache) - informant.collect(:sidekiq, :sidekiq) if ENV['STATSD_SIDEKIQ'] == 'true' + NSA.inform_statsd(statsd) do |informant| + informant.collect(:action_controller, :web) + informant.collect(:active_record, :db) + informant.collect(:active_support_cache, :cache) + informant.collect(:sidekiq, :sidekiq) if ENV['STATSD_SIDEKIQ'] == 'true' + end + rescue + Rails.logger.warn("statsd address #{ENV['STATSD_ADDR']} not reachable, proceeding without statsd") end end From 846f59c6e95ae0f6ccd5714bd1a57c8dd0fed017 Mon Sep 17 00:00:00 2001 From: David Roetzel Date: Thu, 27 Jun 2024 16:40:19 +0200 Subject: [PATCH 10/17] Add size limit for link preview URLs (#30854) --- app/services/fetch_link_card_service.rb | 5 ++++- spec/services/fetch_link_card_service_spec.rb | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index 13775e63c1..8302a9ef47 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -15,6 +15,9 @@ class FetchLinkCardService < BaseService ) }iox + # URL size limit to safely store in PosgreSQL's unique indexes + BYTESIZE_LIMIT = 2692 + def call(status) @status = status @original_url = parse_urls @@ -85,7 +88,7 @@ class FetchLinkCardService < BaseService def bad_url?(uri) # Avoid local instance URLs and invalid URLs - uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme) + uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme) || uri.to_s.bytesize > BYTESIZE_LIMIT end def mention_link?(anchor) diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb index f44cbb750c..fa4d7bb366 100644 --- a/spec/services/fetch_link_card_service_spec.rb +++ b/spec/services/fetch_link_card_service_spec.rb @@ -184,6 +184,19 @@ RSpec.describe FetchLinkCardService, type: :service do end end + context 'with an URL too long for PostgreSQL unique indexes' do + let(:url) { "http://example.com/#{'a' * 2674}" } + let(:status) { Fabricate(:status, text: url) } + + it 'does not fetch the URL' do + expect(a_request(:get, url)).to_not have_been_made + end + + it 'does not create a preview card' do + expect(status.preview_card).to be_nil + end + end + context 'with a URL of a page with oEmbed support' do let(:html) { 'Hello world' } let(:status) { Fabricate(:status, text: 'http://example.com/html') } From 88b2d6eca57e4f8d90062d9eb9d511213bb00a1c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 27 Jun 2024 23:34:34 +0200 Subject: [PATCH 11/17] Change search modifiers to be case-insensitive (#30865) --- app/lib/search_query_transformer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb index a45ae3d09b..8849acf12c 100644 --- a/app/lib/search_query_transformer.rb +++ b/app/lib/search_query_transformer.rb @@ -223,7 +223,7 @@ class SearchQueryTransformer < Parslet::Transform end rule(clause: subtree(:clause)) do - prefix = clause[:prefix][:term].to_s if clause[:prefix] + prefix = clause[:prefix][:term].to_s.downcase if clause[:prefix] operator = clause[:operator]&.to_s term = clause[:phrase] ? clause[:phrase].map { |term| term[:term].to_s }.join(' ') : clause[:term].to_s From 9b6219c48f4fafacc1f31c0f8d89f98948a46aa9 Mon Sep 17 00:00:00 2001 From: David Roetzel Date: Fri, 21 Jun 2024 14:51:10 +0200 Subject: [PATCH 12/17] Improve encoding detection for link cards (#30780) --- app/lib/link_details_extractor.rb | 15 ++++++++++----- .../fixtures/requests/low_confidence_latin1.txt | 17 +++++++++++++++++ spec/services/fetch_link_card_service_spec.rb | 9 +++++++++ 3 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 spec/fixtures/requests/low_confidence_latin1.txt diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb index bb031986d6..48a592a404 100644 --- a/app/lib/link_details_extractor.rb +++ b/app/lib/link_details_extractor.rb @@ -265,16 +265,21 @@ class LinkDetailsExtractor end def document - @document ||= Nokogiri::HTML(@html, nil, encoding) + @document ||= detect_encoding_and_parse_document end - def encoding - @encoding ||= begin - guess = detector.detect(@html, @html_charset) - guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil + def detect_encoding_and_parse_document + [detect_encoding, nil, @html_charset, 'UTF-8'].uniq.each do |encoding| + document = Nokogiri::HTML(@html, nil, encoding) + return document if document.to_s.valid_encoding? end end + def detect_encoding + guess = detector.detect(@html, @html_charset) + guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil + end + def detector @detector ||= CharlockHolmes::EncodingDetector.new.tap do |detector| detector.strip_tags = true diff --git a/spec/fixtures/requests/low_confidence_latin1.txt b/spec/fixtures/requests/low_confidence_latin1.txt new file mode 100644 index 0000000000..39c3e23d64 --- /dev/null +++ b/spec/fixtures/requests/low_confidence_latin1.txt @@ -0,0 +1,17 @@ +HTTP/1.1 200 OK +server: nginx +date: Thu, 13 Jun 2024 14:33:13 GMT +content-type: text/html; charset=ISO-8859-1 +content-length: 158 +accept-ranges: bytes + + + + + + Tofu á l'orange + + +

Tofu á l'orange

+ + diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb index fa4d7bb366..592dce24d4 100644 --- a/spec/services/fetch_link_card_service_spec.rb +++ b/spec/services/fetch_link_card_service_spec.rb @@ -26,6 +26,7 @@ RSpec.describe FetchLinkCardService, type: :service do stub_request(:get, 'http://example.com/sjis_with_wrong_charset').to_return(request_fixture('sjis_with_wrong_charset.txt')) stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt')) stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt')) + stub_request(:get, 'http://example.com/low_confidence_latin1').to_return(request_fixture('low_confidence_latin1.txt')) Rails.cache.write('oembed_endpoint:example.com', oembed_cache) if oembed_cache @@ -148,6 +149,14 @@ RSpec.describe FetchLinkCardService, type: :service do end end + context 'with a URL of a page in ISO-8859-1 encoding, that charlock_holmes cannot detect' do + let(:status) { Fabricate(:status, text: 'Check out http://example.com/low_confidence_latin1') } + + it 'decodes the HTML' do + expect(status.preview_card.title).to eq("Tofu á l'orange") + end + end + context 'with a Japanese path URL' do let(:status) { Fabricate(:status, text: 'テストhttp://example.com/日本語') } From 6cd9bd6ae154eee02c42121ffaae280de907edbb Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 3 Jul 2024 09:15:47 +0200 Subject: [PATCH 13/17] fix: Return HTTP 422 when scheduled status time is less than 5 minutes (#30584) --- app/services/post_status_service.rb | 2 +- .../api/v1/statuses_controller_spec.rb | 40 +++++++++++++++++++ spec/services/post_status_service_spec.rb | 10 +++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index e4dd480f10..280666dcf7 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -171,7 +171,7 @@ class PostStatusService < BaseService end def scheduled_in_the_past? - @scheduled_at.present? && @scheduled_at <= Time.now.utc + MIN_SCHEDULE_OFFSET + @scheduled_at.present? && @scheduled_at <= Time.now.utc end def bump_potential_friendship! diff --git a/spec/controllers/api/v1/statuses_controller_spec.rb b/spec/controllers/api/v1/statuses_controller_spec.rb index c2bdba9ace..591307fe22 100644 --- a/spec/controllers/api/v1/statuses_controller_spec.rb +++ b/spec/controllers/api/v1/statuses_controller_spec.rb @@ -182,6 +182,46 @@ RSpec.describe Api::V1::StatusesController do expect(response.headers['X-RateLimit-Remaining']).to eq '0' end end + + context 'with missing thread' do + subject { post :create, params: params } + + let(:params) { { status: 'Hello world', in_reply_to_id: 0 } } + + it 'returns http not found' do + subject + + expect(response).to have_http_status(404) + end + end + + context 'when scheduling a status' do + subject { post :create, params: params } + + let(:params) { { status: 'Hello world', scheduled_at: 10.minutes.from_now } } + let(:account) { user.account } + + it 'returns HTTP 200' do + subject + + expect(response).to have_http_status(200) + end + + it 'creates a scheduled status' do + expect { subject }.to change { account.scheduled_statuses.count }.from(0).to(1) + end + + context 'when the scheduling time is less than 5 minutes' do + let(:params) { { status: 'Hello world', scheduled_at: 4.minutes.from_now } } + + it 'does not create a scheduled status', :aggregate_failures do + subject + + expect(response).to have_http_status(422) + expect(account.scheduled_statuses).to be_empty + end + end + end end describe 'DELETE #destroy' do diff --git a/spec/services/post_status_service_spec.rb b/spec/services/post_status_service_spec.rb index a74e826261..826af54794 100644 --- a/spec/services/post_status_service_spec.rb +++ b/spec/services/post_status_service_spec.rb @@ -61,6 +61,16 @@ RSpec.describe PostStatusService, type: :service do status2 = subject.call(account, text: 'test', idempotency: 'meepmeep', scheduled_at: future) expect(status2.id).to eq status1.id end + + context 'when scheduled_at is less than min offset' do + let(:invalid_scheduled_time) { 4.minutes.from_now } + + it 'raises invalid record error' do + expect do + subject.call(account, text: 'Hi future!', scheduled_at: invalid_scheduled_time) + end.to raise_error(ActiveRecord::RecordInvalid) + end + end end it 'creates response to the original status of boost' do From df974a912b59e276ad7bfa413e2b628633a776d4 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 4 Jul 2024 16:11:28 +0200 Subject: [PATCH 14/17] Merge pull request from GHSA-vp5r-5pgw-jwqx * Fix streaming sessions not being closed when revoking access to an app * Add tests for GHSA-7w3c-p9j8-mq3x --- .../oauth/authorized_applications_controller.rb | 1 + app/lib/application_extension.rb | 8 +++++--- .../oauth/authorized_applications_controller_spec.rb | 10 ++++++++++ .../settings/applications_controller_spec.rb | 8 ++++++++ 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/app/controllers/oauth/authorized_applications_controller.rb b/app/controllers/oauth/authorized_applications_controller.rb index 350ae2e906..17f1be23de 100644 --- a/app/controllers/oauth/authorized_applications_controller.rb +++ b/app/controllers/oauth/authorized_applications_controller.rb @@ -17,6 +17,7 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio def destroy Web::PushSubscription.unsubscribe_for(params[:id], current_resource_owner) + Doorkeeper::Application.find_by(id: params[:id])&.close_streaming_sessions(current_resource_owner) super end diff --git a/app/lib/application_extension.rb b/app/lib/application_extension.rb index 400c51a023..f226b99cd7 100644 --- a/app/lib/application_extension.rb +++ b/app/lib/application_extension.rb @@ -16,17 +16,19 @@ module ApplicationExtension # dependent: delete_all, which means the ActiveRecord callback in # AccessTokenExtension is not run, so instead we manually announce to # streaming that these tokens are being deleted. - before_destroy :push_to_streaming_api, prepend: true + before_destroy :close_streaming_sessions, prepend: true end def confirmation_redirect_uri redirect_uri.lines.first.strip end - def push_to_streaming_api + def close_streaming_sessions(resource_owner = nil) # TODO: #28793 Combine into a single topic payload = Oj.dump(event: :kill) - access_tokens.in_batches do |tokens| + scope = access_tokens + scope = scope.where(resource_owner_id: resource_owner.id) unless resource_owner.nil? + scope.in_batches do |tokens| redis.pipelined do |pipeline| tokens.ids.each do |id| pipeline.publish("timeline:access_token:#{id}", payload) diff --git a/spec/controllers/oauth/authorized_applications_controller_spec.rb b/spec/controllers/oauth/authorized_applications_controller_spec.rb index b54610604c..3fd9f9499f 100644 --- a/spec/controllers/oauth/authorized_applications_controller_spec.rb +++ b/spec/controllers/oauth/authorized_applications_controller_spec.rb @@ -50,9 +50,11 @@ describe Oauth::AuthorizedApplicationsController do let!(:application) { Fabricate(:application) } let!(:access_token) { Fabricate(:accessible_access_token, application: application, resource_owner_id: user.id) } let!(:web_push_subscription) { Fabricate(:web_push_subscription, user: user, access_token: access_token) } + let(:redis_pipeline_stub) { instance_double(Redis::Namespace, publish: nil) } before do sign_in user, scope: :user + allow(redis).to receive(:pipelined).and_yield(redis_pipeline_stub) post :destroy, params: { id: application.id } end @@ -63,5 +65,13 @@ describe Oauth::AuthorizedApplicationsController do it 'removes subscriptions for the application\'s access tokens' do expect(Web::PushSubscription.where(user: user).count).to eq 0 end + + it 'removes the web_push_subscription' do + expect { web_push_subscription.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + + it 'sends a session kill payload to the streaming server' do + expect(redis_pipeline_stub).to have_received(:publish).with("timeline:access_token:#{access_token.id}", '{"event":"kill"}') + end end end diff --git a/spec/controllers/settings/applications_controller_spec.rb b/spec/controllers/settings/applications_controller_spec.rb index 169304b3ed..88974e551d 100644 --- a/spec/controllers/settings/applications_controller_spec.rb +++ b/spec/controllers/settings/applications_controller_spec.rb @@ -166,7 +166,11 @@ describe Settings::ApplicationsController do end describe 'destroy' do + let(:redis_pipeline_stub) { instance_double(Redis::Namespace, publish: nil) } + let!(:access_token) { Fabricate(:accessible_access_token, application: app) } + before do + allow(redis).to receive(:pipelined).and_yield(redis_pipeline_stub) post :destroy, params: { id: app.id } end @@ -177,6 +181,10 @@ describe Settings::ApplicationsController do it 'removes the app' do expect(Doorkeeper::Application.find_by(id: app.id)).to be_nil end + + it 'sends a session kill payload to the streaming server' do + expect(redis_pipeline_stub).to have_received(:publish).with("timeline:access_token:#{access_token.id}", '{"event":"kill"}') + end end describe 'regenerate' do From 4fb4721072e552dc2fa7541f7bdeb1737a6c113e Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 4 Jul 2024 16:26:49 +0200 Subject: [PATCH 15/17] Merge pull request from GHSA-58x8-3qxw-6hm7 * Fix insufficient permission checking for public timeline endpoints Note that this changes unauthenticated access failure code from 401 to 422 * Add more tests for public timelines * Require user token in `/api/v1/statuses/:id/translate` and `/api/v1/scheduled_statuses` --- .../api/v1/scheduled_statuses_controller.rb | 1 + .../v1/statuses/translations_controller.rb | 1 + .../api/v1/timelines/public_controller.rb | 1 + .../api/v1/timelines/tag_controller.rb | 3 ++- .../v1/scheduled_statuses_controller_spec.rb | 11 ++++++++ .../statuses/translations_controller_spec.rb | 20 ++++++++++++++ .../api/v1/timelines/tag_controller_spec.rb | 19 +++++++++++--- spec/requests/api/v1/timelines/public_spec.rb | 26 ++++++++++++++----- 8 files changed, 71 insertions(+), 11 deletions(-) diff --git a/app/controllers/api/v1/scheduled_statuses_controller.rb b/app/controllers/api/v1/scheduled_statuses_controller.rb index 2220b6d22e..b33b534ebb 100644 --- a/app/controllers/api/v1/scheduled_statuses_controller.rb +++ b/app/controllers/api/v1/scheduled_statuses_controller.rb @@ -6,6 +6,7 @@ class Api::V1::ScheduledStatusesController < Api::BaseController before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, except: [:update, :destroy] before_action -> { doorkeeper_authorize! :write, :'write:statuses' }, only: [:update, :destroy] + before_action :require_user! before_action :set_statuses, only: :index before_action :set_status, except: :index diff --git a/app/controllers/api/v1/statuses/translations_controller.rb b/app/controllers/api/v1/statuses/translations_controller.rb index ec5ea5b85b..5e5ee7d38e 100644 --- a/app/controllers/api/v1/statuses/translations_controller.rb +++ b/app/controllers/api/v1/statuses/translations_controller.rb @@ -4,6 +4,7 @@ class Api::V1::Statuses::TranslationsController < Api::BaseController include Authorization before_action -> { doorkeeper_authorize! :read, :'read:statuses' } + before_action :require_user! before_action :set_status before_action :set_translation diff --git a/app/controllers/api/v1/timelines/public_controller.rb b/app/controllers/api/v1/timelines/public_controller.rb index 5bbd92b9ee..0ff2c5aee2 100644 --- a/app/controllers/api/v1/timelines/public_controller.rb +++ b/app/controllers/api/v1/timelines/public_controller.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true class Api::V1::Timelines::PublicController < Api::BaseController + before_action -> { authorize_if_got_token! :read, :'read:statuses' } before_action :require_user!, only: [:show], if: :require_auth? after_action :insert_pagination_headers, unless: -> { @statuses.empty? } diff --git a/app/controllers/api/v1/timelines/tag_controller.rb b/app/controllers/api/v1/timelines/tag_controller.rb index a79d65c124..ffea89c258 100644 --- a/app/controllers/api/v1/timelines/tag_controller.rb +++ b/app/controllers/api/v1/timelines/tag_controller.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true class Api::V1::Timelines::TagController < Api::BaseController - before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: :show, if: :require_auth? + before_action -> { authorize_if_got_token! :read, :'read:statuses' } + before_action :require_user!, if: :require_auth? before_action :load_tag after_action :insert_pagination_headers, unless: -> { @statuses.empty? } diff --git a/spec/controllers/api/v1/scheduled_statuses_controller_spec.rb b/spec/controllers/api/v1/scheduled_statuses_controller_spec.rb index 256c4b272a..cc3b65f37d 100644 --- a/spec/controllers/api/v1/scheduled_statuses_controller_spec.rb +++ b/spec/controllers/api/v1/scheduled_statuses_controller_spec.rb @@ -13,6 +13,17 @@ describe Api::V1::ScheduledStatusesController do allow(controller).to receive(:doorkeeper_token) { token } end + context 'with an application token' do + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil, scopes: 'read:statuses') } + + it 'returns http unprocessable entity' do + get :index + + expect(response) + .to have_http_status(422) + end + end + describe 'GET #index' do it 'returns http success' do get :index diff --git a/spec/controllers/api/v1/statuses/translations_controller_spec.rb b/spec/controllers/api/v1/statuses/translations_controller_spec.rb index 6257494ae1..da152843b2 100644 --- a/spec/controllers/api/v1/statuses/translations_controller_spec.rb +++ b/spec/controllers/api/v1/statuses/translations_controller_spec.rb @@ -9,6 +9,26 @@ describe Api::V1::Statuses::TranslationsController do let(:app) { Fabricate(:application, name: 'Test app', website: 'http://testapp.com') } let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses', application: app) } + context 'with an application token' do + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil, scopes: 'read:statuses', application: app) } + + before do + allow(controller).to receive(:doorkeeper_token) { token } + end + + describe 'POST /api/v1/statuses/:status_id/translate' do + let(:status) { Fabricate(:status, account: user.account, text: 'Hola', language: 'es') } + + before do + post :create, params: { status_id: status.id } + end + + it 'returns http unprocessable entity' do + expect(response).to have_http_status(422) + end + end + end + context 'with an oauth token' do before do allow(controller).to receive(:doorkeeper_token) { token } diff --git a/spec/controllers/api/v1/timelines/tag_controller_spec.rb b/spec/controllers/api/v1/timelines/tag_controller_spec.rb index 1c60798fcf..89622a41a2 100644 --- a/spec/controllers/api/v1/timelines/tag_controller_spec.rb +++ b/spec/controllers/api/v1/timelines/tag_controller_spec.rb @@ -6,7 +6,8 @@ describe Api::V1::Timelines::TagController do render_views let(:user) { Fabricate(:user) } - let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses') } + let(:scopes) { 'read:statuses' } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) } before do allow(controller).to receive(:doorkeeper_token) { token } @@ -48,13 +49,23 @@ describe Api::V1::Timelines::TagController do Form::AdminSettings.new(timeline_preview: false).save end - context 'when the user is not authenticated' do + context 'without an access token' do let(:token) { nil } - it 'returns http unauthorized' do + it 'returns http unprocessable entity' do subject - expect(response).to have_http_status(401) + expect(response).to have_http_status(422) + end + end + + context 'with an application access token, not bound to a user' do + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil, scopes: scopes) } + + it 'returns http unprocessable entity' do + subject + + expect(response).to have_http_status(422) end end diff --git a/spec/requests/api/v1/timelines/public_spec.rb b/spec/requests/api/v1/timelines/public_spec.rb index c436262407..03bde3da87 100644 --- a/spec/requests/api/v1/timelines/public_spec.rb +++ b/spec/requests/api/v1/timelines/public_spec.rb @@ -32,6 +32,8 @@ describe 'Public' do context 'when the instance allows public preview' do let(:expected_statuses) { [local_status, remote_status, media_status] } + it_behaves_like 'forbidden for wrong scope', 'profile' + context 'with an authorized user' do it_behaves_like 'a successful request to the public timeline' end @@ -96,13 +98,9 @@ describe 'Public' do Form::AdminSettings.new(timeline_preview: false).save end - context 'with an authenticated user' do - let(:expected_statuses) { [local_status, remote_status, media_status] } + it_behaves_like 'forbidden for wrong scope', 'profile' - it_behaves_like 'a successful request to the public timeline' - end - - context 'with an unauthenticated user' do + context 'without an authentication token' do let(:headers) { {} } it 'returns http unprocessable entity' do @@ -111,6 +109,22 @@ describe 'Public' do expect(response).to have_http_status(422) end end + + context 'with an application access token, not bound to a user' do + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil, scopes: scopes) } + + it 'returns http unprocessable entity' do + subject + + expect(response).to have_http_status(422) + end + end + + context 'with an authenticated user' do + let(:expected_statuses) { [local_status, remote_status, media_status] } + + it_behaves_like 'a successful request to the public timeline' + end end end end From d4bf22b632ea8b1174375c4966a6768ab66393b6 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 4 Jul 2024 16:45:52 +0200 Subject: [PATCH 16/17] Merge pull request from GHSA-xjvf-fm67-4qc3 --- app/lib/activitypub/activity/create.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index fedfa39dee..1581555bde 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -104,7 +104,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def find_existing_status status = status_from_uri(object_uri) status ||= Status.find_by(uri: @object['atomUri']) if @object['atomUri'].present? - status + status if status&.account_id == @account.id end def process_status_params From a5b4a2b7e71aedd2f04bc7a90f79dfe234fa7f89 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 4 Jul 2024 16:46:35 +0200 Subject: [PATCH 17/17] Bump version to v4.2.10 (#30910) --- CHANGELOG.md | 31 +++++++++++++++++++++++++++++++ docker-compose.yml | 6 +++--- lib/mastodon/version.rb | 2 +- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e579e1489..e57d5f23b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,37 @@ All notable changes to this project will be documented in this file. +## [4.2.10] - 2024-07-04 + +### Security + +- Fix incorrect permission checking on multiple API endpoints ([GHSA-58x8-3qxw-6hm7](https://github.com/mastodon/mastodon/security/advisories/GHSA-58x8-3qxw-6hm7)) +- Fix incorrect authorship checking when processing some activities (CVE-2024-37903, [GHSA-xjvf-fm67-4qc3](https://github.com/mastodon/mastodon/security/advisories/GHSA-xjvf-fm67-4qc3)) +- Fix ongoing streaming sessions not being invalidated when application tokens get revoked ([GHSA-vp5r-5pgw-jwqx](https://github.com/mastodon/mastodon/security/advisories/GHSA-vp5r-5pgw-jwqx)) +- Update dependencies + +### Added + +- Add yarn version specification to avoid confusion with Yarn 3 and Yarn 4 + +### Changed + +- Change preview cards generation to skip unusually long URLs ([oneiros](https://github.com/mastodon/mastodon/pull/30854)) +- Change search modifiers to be case-insensitive ([Gargron](https://github.com/mastodon/mastodon/pull/30865)) +- Change `STATSD_ADDR` handling to emit a warning rather than crashing if the address is unreachable ([timothyjrogers](https://github.com/mastodon/mastodon/pull/30691)) +- Change PWA start URL from `/home` to `/` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27377)) + +### Removed + +- Removed dependency on `posix-spawn` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/18559)) + +### Fixed + +- Fix scheduled statuses scheduled in less than 5 minutes being immediately published ([danielmbrasil](https://github.com/mastodon/mastodon/pull/30584)) +- Fix encoding detection for link cards ([oneiros](https://github.com/mastodon/mastodon/pull/30780)) +- Fix `/admin/accounts/:account_id/statuses/:id` for edited posts with media attachments ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30819)) +- Fix duplicate `@context` attribute in user archive export ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30653)) + ## [4.2.9] - 2024-05-30 ### Security diff --git a/docker-compose.yml b/docker-compose.yml index 10a57e31e3..d61aa05582 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,7 +56,7 @@ services: web: build: . - image: ghcr.io/mastodon/mastodon:v4.2.9 + image: ghcr.io/mastodon/mastodon:v4.2.10 restart: always env_file: .env.production command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" @@ -77,7 +77,7 @@ services: streaming: build: . - image: ghcr.io/mastodon/mastodon:v4.2.9 + image: ghcr.io/mastodon/mastodon:v4.2.10 restart: always env_file: .env.production command: node ./streaming @@ -95,7 +95,7 @@ services: sidekiq: build: . - image: ghcr.io/mastodon/mastodon:v4.2.9 + image: ghcr.io/mastodon/mastodon:v4.2.10 restart: always env_file: .env.production command: bundle exec sidekiq diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index f9088382f8..77e85a1652 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 9 + 10 end def default_prerelease