Add delivery failure handling to FASP jobs (#35723)

spec/models/fasp/provider_spec.rb

@@ -214,4 +214,102 @@ RSpec.describe Fasp::Provider do
       expect(subject.delivery_failure_tracker).to be_a(DeliveryFailureTracker)
     end
   end
+
+  describe '#available?' do
+    subject { Fabricate(:fasp_provider, delivery_last_failed_at:) }
+
+    let(:delivery_last_failed_at) { nil }
+
+    before do
+      allow(subject.delivery_failure_tracker).to receive(:available?).and_return(available)
+    end
+
+    context 'when the delivery failure tracker reports it is available' do
+      let(:available) { true }
+
+      it 'returns true' do
+        expect(subject.available?).to be true
+      end
+    end
+
+    context 'when the delivery failure tracker reports it is unavailable' do
+      let(:available) { false }
+
+      context 'when the last failure was more than one hour ago' do
+        let(:delivery_last_failed_at) { 61.minutes.ago }
+
+        it 'returns true' do
+          expect(subject.available?).to be true
+        end
+      end
+
+      context 'when the last failure is very recent' do
+        let(:delivery_last_failed_at) { 5.minutes.ago }
+
+        it 'returns false' do
+          expect(subject.available?).to be false
+        end
+      end
+    end
+  end
+
+  describe '#update_availability!' do
+    subject { Fabricate(:fasp_provider, delivery_last_failed_at:) }
+
+    before do
+      allow(subject.delivery_failure_tracker).to receive(:available?).and_return(available)
+    end
+
+    context 'when `delivery_last_failed_at` is `nil`' do
+      let(:delivery_last_failed_at) { nil }
+
+      context 'when the delivery failure tracker reports it is available' do
+        let(:available) { true }
+
+        it 'does not update the provider' do
+          subject.update_availability!
+
+          expect(subject.saved_changes?).to be false
+        end
+      end
+
+      context 'when the delivery failure tracker reports it is unavailable' do
+        let(:available) { false }
+
+        it 'sets `delivery_last_failed_at` to the current time' do
+          freeze_time
+
+          subject.update_availability!
+
+          expect(subject.delivery_last_failed_at).to eq Time.zone.now
+        end
+      end
+    end
+
+    context 'when `delivery_last_failed_at` is present' do
+      context 'when the delivery failure tracker reports it is available' do
+        let(:available) { true }
+        let(:delivery_last_failed_at) { 5.minutes.ago }
+
+        it 'sets `delivery_last_failed_at` to `nil`' do
+          subject.update_availability!
+
+          expect(subject.delivery_last_failed_at).to be_nil
+        end
+      end
+
+      context 'when the delivery failure tracker reports it is unavailable' do
+        let(:available) { false }
+        let(:delivery_last_failed_at) { 5.minutes.ago }
+
+        it 'updates `delivery_last_failed_at` to the current time' do
+          freeze_time
+
+          subject.update_availability!
+
+          expect(subject.delivery_last_failed_at).to eq Time.zone.now
+        end
+      end
+    end
+  end
 end

spec/support/examples/workers/fasp/delivery_failure.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+RSpec.shared_examples 'worker handling fasp delivery failures' do
+  context 'when provider is not available' do
+    before do
+      provider.update(delivery_last_failed_at: 1.minute.ago)
+      domain = Addressable::URI.parse(provider.base_url).normalized_host
+      UnavailableDomain.create!(domain:)
+    end
+
+    it 'does not attempt connecting and does not fail the job' do
+      expect { subject }.to_not raise_error
+      expect(stubbed_request).to_not have_been_made
+    end
+  end
+
+  context 'when connection to provider fails' do
+    before do
+      base_stubbed_request
+        .to_raise(HTTP::ConnectionError)
+    end
+
+    context 'when provider becomes unavailable' do
+      before do
+        travel_to 5.minutes.ago
+        4.times do
+          provider.delivery_failure_tracker.track_failure!
+          travel_to 1.minute.since
+        end
+      end
+
+      it 'updates the provider and does not fail the job, so it will not be retried' do
+        expect { subject }.to_not raise_error
+        expect(provider.reload.delivery_last_failed_at).to eq Time.current
+      end
+    end
+
+    context 'when provider is still marked as available' do
+      it 'fails the job so it can be retried' do
+        expect { subject }.to raise_error(HTTP::ConnectionError)
+      end
+    end
+  end
+
+  context 'when connection to a previously unavailable provider succeeds' do
+    before do
+      provider.update(delivery_last_failed_at: 2.hours.ago)
+      domain = Addressable::URI.parse(provider.base_url).normalized_host
+      UnavailableDomain.create!(domain:)
+    end
+
+    it 'marks the provider as being available again' do
+      expect { subject }.to_not raise_error
+      expect(provider).to be_available
+    end
+  end
+end

spec/workers/fasp/account_search_worker_spec.rb

@@ -5,12 +5,14 @@ require 'rails_helper'
 RSpec.describe Fasp::AccountSearchWorker, feature: :fasp do
   include ProviderRequestHelper
 
+  subject { described_class.new.perform('cats') }
+
   let(:provider) { Fabricate(:account_search_fasp) }
   let(:account) { Fabricate(:account) }
   let(:fetch_service) { instance_double(ActivityPub::FetchRemoteActorService, call: true) }
+  let(:path) { '/account_search/v0/search?term=cats&limit=10' }
 
   let!(:stubbed_request) do
-    path = '/account_search/v0/search?term=cats&limit=10'
     stub_provider_request(provider,
                           method: :get,
                           path:,
@@ -25,7 +27,7 @@ RSpec.describe Fasp::AccountSearchWorker, feature: :fasp do
   end
 
   it 'requests search results and fetches received account uris' do
-    described_class.new.perform('cats')
+    subject
 
     expect(stubbed_request).to have_been_made
     expect(fetch_service).to have_received(:call).with('https://fedi.example.com/accounts/2')
@@ -35,7 +37,7 @@ RSpec.describe Fasp::AccountSearchWorker, feature: :fasp do
   it 'marks a running async refresh as finished' do
     async_refresh = AsyncRefresh.create("fasp:account_search:#{Digest::MD5.base64digest('cats')}", count_results: true)
 
-    described_class.new.perform('cats')
+    subject
 
     expect(async_refresh.reload).to be_finished
   end
@@ -43,8 +45,16 @@ RSpec.describe Fasp::AccountSearchWorker, feature: :fasp do
   it 'tracks the number of fetched accounts in the async refresh' do
     async_refresh = AsyncRefresh.create("fasp:account_search:#{Digest::MD5.base64digest('cats')}", count_results: true)
 
-    described_class.new.perform('cats')
+    subject
 
     expect(async_refresh.reload.result_count).to eq 2
   end
+
+  describe 'provider delivery failure handling' do
+    let(:base_stubbed_request) do
+      stub_request(:get, provider.url(path))
+    end
+
+    it_behaves_like('worker handling fasp delivery failures')
+  end
 end

spec/workers/fasp/announce_account_lifecycle_event_worker_spec.rb

@@ -5,15 +5,19 @@ require 'rails_helper'
 RSpec.describe Fasp::AnnounceAccountLifecycleEventWorker do
   include ProviderRequestHelper
 
+  subject { described_class.new.perform(account_uri, 'new') }
+
   let(:account_uri) { 'https://masto.example.com/accounts/1' }
   let(:subscription) do
     Fabricate(:fasp_subscription, category: 'account')
   end
   let(:provider) { subscription.fasp_provider }
+  let(:path) { '/data_sharing/v0/announcements' }
+
   let!(:stubbed_request) do
     stub_provider_request(provider,
                           method: :post,
-                          path: '/data_sharing/v0/announcements',
+                          path:,
                           response_body: {
                             source: {
                               subscription: {
@@ -27,8 +31,16 @@ RSpec.describe Fasp::AnnounceAccountLifecycleEventWorker do
   end
 
   it 'sends the account uri to subscribed providers' do
-    described_class.new.perform(account_uri, 'new')
+    subject
 
     expect(stubbed_request).to have_been_made
   end
+
+  describe 'provider delivery failure handling' do
+    let(:base_stubbed_request) do
+      stub_request(:post, provider.url(path))
+    end
+
+    it_behaves_like('worker handling fasp delivery failures')
+  end
 end

spec/workers/fasp/announce_content_lifecycle_event_worker_spec.rb

@@ -5,15 +5,19 @@ require 'rails_helper'
 RSpec.describe Fasp::AnnounceContentLifecycleEventWorker do
   include ProviderRequestHelper
 
+  subject { described_class.new.perform(status_uri, 'new') }
+
   let(:status_uri) { 'https://masto.example.com/status/1' }
   let(:subscription) do
     Fabricate(:fasp_subscription)
   end
   let(:provider) { subscription.fasp_provider }
+  let(:path) { '/data_sharing/v0/announcements' }
+
   let!(:stubbed_request) do
     stub_provider_request(provider,
                           method: :post,
-                          path: '/data_sharing/v0/announcements',
+                          path:,
                           response_body: {
                             source: {
                               subscription: {
@@ -27,8 +31,16 @@ RSpec.describe Fasp::AnnounceContentLifecycleEventWorker do
   end
 
   it 'sends the status uri to subscribed providers' do
-    described_class.new.perform(status_uri, 'new')
+    subject
 
     expect(stubbed_request).to have_been_made
   end
+
+  describe 'provider delivery failure handling' do
+    let(:base_stubbed_request) do
+      stub_request(:post, provider.url(path))
+    end
+
+    it_behaves_like('worker handling fasp delivery failures')
+  end
 end

spec/workers/fasp/announce_trend_worker_spec.rb

@@ -5,6 +5,8 @@ require 'rails_helper'
 RSpec.describe Fasp::AnnounceTrendWorker do
   include ProviderRequestHelper
 
+  subject { described_class.new.perform(status.id, 'favourite') }
+
   let(:status) { Fabricate(:status) }
   let(:subscription) do
     Fabricate(:fasp_subscription,
@@ -14,10 +16,12 @@ RSpec.describe Fasp::AnnounceTrendWorker do
               threshold_likes: 2)
   end
   let(:provider) { subscription.fasp_provider }
+  let(:path) { '/data_sharing/v0/announcements' }
+
   let!(:stubbed_request) do
     stub_provider_request(provider,
                           method: :post,
-                          path: '/data_sharing/v0/announcements',
+                          path:,
                           response_body: {
                             source: {
                               subscription: {
@@ -36,15 +40,23 @@ RSpec.describe Fasp::AnnounceTrendWorker do
     end
 
     it 'sends the account uri to subscribed providers' do
-      described_class.new.perform(status.id, 'favourite')
+      subject
 
       expect(stubbed_request).to have_been_made
     end
+
+    describe 'provider delivery failure handling' do
+      let(:base_stubbed_request) do
+        stub_request(:post, provider.url(path))
+      end
+
+      it_behaves_like('worker handling fasp delivery failures')
+    end
   end
 
   context 'when the configured threshold is not met' do
     it 'does not notify any provider' do
-      described_class.new.perform(status.id, 'favourite')
+      subject
 
       expect(stubbed_request).to_not have_been_made
     end

spec/workers/fasp/backfill_worker_spec.rb

@@ -5,13 +5,17 @@ require 'rails_helper'
 RSpec.describe Fasp::BackfillWorker do
   include ProviderRequestHelper
 
+  subject { described_class.new.perform(backfill_request.id) }
+
   let(:backfill_request) { Fabricate(:fasp_backfill_request) }
   let(:provider) { backfill_request.fasp_provider }
   let(:status) { Fabricate(:status) }
+  let(:path) { '/data_sharing/v0/announcements' }
+
   let!(:stubbed_request) do
     stub_provider_request(provider,
                           method: :post,
-                          path: '/data_sharing/v0/announcements',
+                          path:,
                           response_body: {
                             source: {
                               backfillRequest: {
@@ -25,8 +29,16 @@ RSpec.describe Fasp::BackfillWorker do
   end
 
   it 'sends status uri to provider that requested backfill' do
-    described_class.new.perform(backfill_request.id)
+    subject
 
     expect(stubbed_request).to have_been_made
   end
+
+  describe 'provider delivery failure handling' do
+    let(:base_stubbed_request) do
+      stub_request(:post, provider.url(path))
+    end
+
+    it_behaves_like('worker handling fasp delivery failures')
+  end
 end

spec/workers/fasp/follow_recommendation_worker_spec.rb

@@ -5,13 +5,15 @@ require 'rails_helper'
 RSpec.describe Fasp::FollowRecommendationWorker, feature: :fasp do
   include ProviderRequestHelper
 
+  subject { described_class.new.perform(account.id) }
+
   let(:provider) { Fabricate(:follow_recommendation_fasp) }
   let(:account) { Fabricate(:account) }
+  let(:account_uri) { ActivityPub::TagManager.instance.uri_for(account) }
   let(:fetch_service) { instance_double(ActivityPub::FetchRemoteActorService) }
+  let(:path) { "/follow_recommendation/v0/accounts?accountUri=#{URI.encode_uri_component(account_uri)}" }
 
   let!(:stubbed_request) do
-    account_uri = ActivityPub::TagManager.instance.uri_for(account)
-    path = "/follow_recommendation/v0/accounts?accountUri=#{URI.encode_uri_component(account_uri)}"
     stub_provider_request(provider,
                           method: :get,
                           path:,
@@ -28,7 +30,7 @@ RSpec.describe Fasp::FollowRecommendationWorker, feature: :fasp do
   end
 
   it "sends the requesting account's uri to provider and fetches received account uris" do
-    described_class.new.perform(account.id)
+    subject
 
     expect(stubbed_request).to have_been_made
     expect(fetch_service).to have_received(:call).with('https://fedi.example.com/accounts/1')
@@ -38,7 +40,7 @@ RSpec.describe Fasp::FollowRecommendationWorker, feature: :fasp do
   it 'marks a running async refresh as finished' do
     async_refresh = AsyncRefresh.create("fasp:follow_recommendation:#{account.id}", count_results: true)
 
-    described_class.new.perform(account.id)
+    subject
 
     expect(async_refresh.reload).to be_finished
   end
@@ -46,14 +48,22 @@ RSpec.describe Fasp::FollowRecommendationWorker, feature: :fasp do
   it 'tracks the number of fetched accounts in the async refresh' do
     async_refresh = AsyncRefresh.create("fasp:follow_recommendation:#{account.id}", count_results: true)
 
-    described_class.new.perform(account.id)
+    subject
 
     expect(async_refresh.reload.result_count).to eq 2
   end
 
   it 'persists the results' do
     expect do
-      described_class.new.perform(account.id)
+      subject
     end.to change(Fasp::FollowRecommendation, :count).by(2)
   end
+
+  describe 'provider delivery failure handling' do
+    let(:base_stubbed_request) do
+      stub_request(:get, provider.url(path))
+    end
+
+    it_behaves_like('worker handling fasp delivery failures')
+  end
 end