Commit 38b6c34e authored by Eugen Rochko's avatar Eugen Rochko Committed by GitHub

Fix issues with tootctl's parallelization and progress reporting (#12093)

parent b5be067c
...@@ -15,6 +15,11 @@ module Mastodon ...@@ -15,6 +15,11 @@ module Mastodon
end end
def parallelize_with_progress(scope) def parallelize_with_progress(scope)
if options[:concurrency] < 1
say('Cannot run with this concurrency setting, must be at least 1', :red)
exit(1)
end
ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency] ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency]
progress = create_progress_bar(scope.count) progress = create_progress_bar(scope.count)
...@@ -27,17 +32,26 @@ module Mastodon ...@@ -27,17 +32,26 @@ module Mastodon
items.each do |item| items.each do |item|
futures << Concurrent::Future.execute(executor: pool) do futures << Concurrent::Future.execute(executor: pool) do
ActiveRecord::Base.connection_pool.with_connection do begin
begin if !progress.total.nil? && progress.progress + 1 > progress.total
progress.log("Processing #{item.id}") if options[:verbose] # The number of items has changed between start and now,
# since there is no good way to predict the final count from
result = yield(item) # here, just change the progress bar to an indeterminate one
aggregate.increment(result) if result.is_a?(Integer)
rescue => e progress.total = nil
progress.log pastel.red("Error processing #{item.id}: #{e}")
ensure
progress.increment
end end
progress.log("Processing #{item.id}") if options[:verbose]
result = ActiveRecord::Base.connection_pool.with_connection do
yield(item)
end
aggregate.increment(result) if result.is_a?(Integer)
rescue => e
progress.log pastel.red("Error processing #{item.id}: #{e}")
ensure
progress.increment
end end
end end
end end
...@@ -46,7 +60,7 @@ module Mastodon ...@@ -46,7 +60,7 @@ module Mastodon
futures.map(&:value) futures.map(&:value)
end end
progress.finish progress.stop
[total.value, aggregate.value] [total.value, aggregate.value]
end end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment