diff --git a/app/models/log_excerpt.rb b/app/models/log_excerpt.rb index 34a24153..876240e6 100644 --- a/app/models/log_excerpt.rb +++ b/app/models/log_excerpt.rb @@ -12,7 +12,7 @@ class LogExcerpt < ApplicationRecord # (missing or empty objects) so that backfill can resume by max report_id. # Network errors are raised to the caller. def self.capture(report) - return nil unless %r{\Ahttps?://rubyci\.s3\.amazonaws\.com/}.match?(report.server&.uri.to_s) + return nil unless report.server&.rubyci_s3? content = fetch_content(report) excerpt = find_or_initialize_by(report_id: report.id) excerpt.content = content @@ -35,14 +35,19 @@ def self.fetch_content(report) sections.map { |s| truncate_bytes(s, budget + slack) }.join("\n") end - # GET a gzipped text file. Returns nil on 404. S3 serves these either with - # Content-Encoding: gzip (Net::HTTP inflates the body) or as raw gzip bytes. + # GET a gzipped text file. Returns nil when the object cannot be read, which + # covers a plain 404 and an object Intelligent-Tiering has moved to an + # archive tier. Neither is retryable, so both become an empty excerpt rather + # than an error. S3 serves these either with Content-Encoding: gzip + # (Net::HTTP inflates the body) or as raw gzip bytes. def self.fetch_text(uri) - uri = URI(uri) - res = Net::HTTP.start(uri.host, uri.port, open_timeout: 10, read_timeout: 30, use_ssl: uri.scheme == "https") do |h| + # Servers are registered under both http:// and https://; always use TLS. + uri = URI(uri.to_s.sub(%r{\Ahttp://}, "https://")) + res = Net::HTTP.start(uri.host, uri.port, open_timeout: 10, read_timeout: 30, use_ssl: true) do |h| h.get(uri.path) end return nil if res.code == "404" + return nil if res.code == "403" && res.body.to_s.include?("InvalidObjectState") res.value body = res.body begin diff --git a/app/models/report.rb b/app/models/report.rb index 2aefd093..fb374a1d 100644 --- a/app/models/report.rb +++ b/app/models/report.rb @@ -289,7 +289,7 @@ def self.fetch_recent ary = [] Server.order(:id).all.each do |server| puts server.uri - if %r<\Ahttps?://rubyci.s3.amazonaws.com/>.match?(server.uri) + if server.rubyci_s3? t = Time.now n = self.get_reports_rubyci_s3(s3, server) ary << [server.uri, Time.now-t, n] diff --git a/app/models/server.rb b/app/models/server.rb index 93ce5a78..92d59b32 100644 --- a/app/models/server.rb +++ b/app/models/server.rb @@ -3,6 +3,21 @@ class Server < ApplicationRecord validates :uri, :length => { :in => 20..200 } validates :ordinal, :numericality => true, :uniqueness => true + RUBYCI_S3_HOST = 'rubyci.s3.amazonaws.com' + RUBYCI_S3_URI = %r{\Ahttps?://#{Regexp.escape(RUBYCI_S3_HOST)}/} + + # Most rows still carry the http:// form these URIs were first registered + # with, so both schemes have to match. Keep the Ruby and SQL forms together + # or they drift apart and each caller covers a different set of servers. + scope :rubyci_s3, -> { + where("uri LIKE :http OR uri LIKE :https", + http: "http://#{RUBYCI_S3_HOST}/%", https: "https://#{RUBYCI_S3_HOST}/%") + } + + def rubyci_s3? + RUBYCI_S3_URI.match?(uri.to_s) + end + def recent_uri(branch) "#{uri.chomp('/')}/ruby-#{branch}/recent.html" end diff --git a/lib/tasks/log_excerpts.rake b/lib/tasks/log_excerpts.rake index 1f177a08..cb7f22d8 100644 --- a/lib/tasks/log_excerpts.rake +++ b/lib/tasks/log_excerpts.rake @@ -26,6 +26,7 @@ end namespace :log_excerpts do desc "Backfill log excerpts from S3. FROM/TO (UTC date, default last 1 year), START_ID to force resume point, SLEEP seconds between reports (default 0.1), FORCE=1 to refetch reports that already have an excerpt" task :backfill => :environment do + $stdout.sync = true # detached dynos block-buffer, hiding progress until exit from = LogExcerptTaskEnv.time("FROM", 1.year.ago) to = LogExcerptTaskEnv.time("TO", Time.now) interval = LogExcerptTaskEnv.float("SLEEP", 0.1) @@ -35,7 +36,7 @@ namespace :log_excerpts do where(datetime: from..to). where.not(ltsv: nil). where("ltsv NOT LIKE '%result:success%'"). - where("servers.uri LIKE 'https://rubyci.s3.amazonaws.com/%'") + merge(Server.rubyci_s3) # Resume by skipping reports that already have an excerpt. Using the max # captured report_id instead would skip everything, because scan_recent_ltsv @@ -72,6 +73,7 @@ namespace :log_excerpts do desc "Delete log excerpts whose report datetime is older than KEEP_DAYS (default 366) days. DRY_RUN=1 to only count." task :prune => :environment do + $stdout.sync = true keep_days = LogExcerptTaskEnv.integer("KEEP_DAYS", 366) abort "KEEP_DAYS must be at least 1, got #{keep_days}" if keep_days < 1 cutoff = keep_days.days.ago diff --git a/test/models/log_excerpt_test.rb b/test/models/log_excerpt_test.rb index 9fc9ca24..6d005fa7 100644 --- a/test/models/log_excerpt_test.rb +++ b/test/models/log_excerpt_test.rb @@ -101,6 +101,15 @@ def create_report(server, attrs = {}) assert_nil report.reload.log_excerpt end + test "capture handles servers registered under http" do + server = create_server("cap-http", uri: "http://rubyci.s3.amazonaws.com/cap-http/") + report = create_report(server) + stub_fetch_text(->(uri) { "body" }) do + LogExcerpt.capture(report) + end + assert_equal "body\nbody", report.reload.log_excerpt.content + end + test "capture is idempotent per report" do server = create_server("cap-idem") report = create_report(server) diff --git a/test/models/server_test.rb b/test/models/server_test.rb new file mode 100644 index 00000000..b2e32ffa --- /dev/null +++ b/test/models/server_test.rb @@ -0,0 +1,30 @@ +require "test_helper" + +class ServerTest < ActiveSupport::TestCase + @@ordinal = 500 + + def create_server(uri) + Server.create!(name: "srv-#{@@ordinal}", uri: uri, ordinal: (@@ordinal += 1)) + end + + test "rubyci_s3? accepts both schemes" do + assert_predicate create_server("https://rubyci.s3.amazonaws.com/ubuntu/"), :rubyci_s3? + assert_predicate create_server("http://rubyci.s3.amazonaws.com/rhel9"), :rubyci_s3? + end + + test "rubyci_s3? rejects other hosts" do + assert_not create_server("https://example.com/rubyci.s3.amazonaws.com/x").rubyci_s3? + assert_not create_server("https://rubyci.s3.amazonaws.com.evil.test/x").rubyci_s3? + assert_not create_server("http://ci.rvm.jp/chkbuild/logs/").rubyci_s3? + end + + test "rubyci_s3 scope selects the same servers as rubyci_s3?" do + servers = [ + create_server("https://rubyci.s3.amazonaws.com/scope-https/"), + create_server("http://rubyci.s3.amazonaws.com/scope-http/"), + create_server("http://ci.rvm.jp/chkbuild/scope-other/"), + ] + selected = Server.rubyci_s3.where(id: servers.map(&:id)).order(:id).to_a + assert_equal servers.select(&:rubyci_s3?), selected + end +end