|
# frozen_string_literal: true
|
|
|
|
require "net/http"
|
|
require "rubygems/package"
|
|
require "tmpdir"
|
|
require "ripper"
|
|
require "yaml"
|
|
|
|
queue = Queue.new
|
|
done = File.exist?("done.yml") ? YAML.load_file("done.yml") : { "gem_versions" => [], "encodings" => {}}
|
|
done_gem_versions = done["gem_versions"]
|
|
|
|
if ARGV[0] == "top100"
|
|
YAML.load(DATA.read).each do |gem_version|
|
|
queue << [gem_version, URI.parse("https://rubygems.org/gems/#{gem_version}.gem")]
|
|
end
|
|
else
|
|
Gem::SpecFetcher.new.available_specs(:latest).first.each do |source, gems|
|
|
gems.each do |tuple|
|
|
gem_version = File.basename(tuple.spec_name, ".gemspec")
|
|
gem_uri = source.uri.merge("/gems/#{gem_version}.gem")
|
|
|
|
next if done["gem_versions"].include?(gem_version)
|
|
queue << [gem_version, gem_uri]
|
|
end
|
|
end
|
|
end
|
|
|
|
encodings = Hash.new { |hash, key| hash[key] = 0 }
|
|
encodings.merge!(done["encodings"])
|
|
|
|
aborted = false
|
|
trap("SIGINT") { aborted = true }
|
|
|
|
Thread.abort_on_exception = true
|
|
ENV.fetch("WORKERS") { 16 }.to_i.times.map do
|
|
Thread.new do
|
|
while !aborted && !queue.empty?
|
|
(gem_version, gem_uri) = queue.shift
|
|
|
|
response =
|
|
begin
|
|
Net::HTTP.get_response(gem_uri)
|
|
rescue Socket::ResolutionError, Errno::ECONNRESET
|
|
retry
|
|
end
|
|
|
|
next unless response.is_a?(Net::HTTPSuccess)
|
|
|
|
Dir.mktmpdir do |directory|
|
|
filepath = File.join(directory, "#{gem_version}.gem")
|
|
File.write(filepath, response.body)
|
|
|
|
begin
|
|
Gem::Package.new(filepath).extract_files(directory, "[!~]*")
|
|
rescue
|
|
next
|
|
end
|
|
|
|
Dir[File.join(directory, "**", "*.rb")].each do |filepath|
|
|
source =
|
|
begin
|
|
File.foreach(filepath).take(2).join
|
|
rescue
|
|
next
|
|
end
|
|
|
|
encoding =
|
|
begin
|
|
Ripper.new(source).tap(&:parse).encoding
|
|
rescue ArgumentError => error
|
|
if error.message =~ /unknown encoding name: (.*)/
|
|
$1
|
|
else
|
|
raise
|
|
end
|
|
rescue
|
|
next
|
|
end
|
|
|
|
encodings[encoding] += 1
|
|
end
|
|
end
|
|
|
|
done_gem_versions << gem_version
|
|
end
|
|
end
|
|
end.each(&:join)
|
|
|
|
if aborted
|
|
File.write("done.yml", YAML.dump("gem_versions" => done_gem_versions.sort, "encodings" => encodings))
|
|
else
|
|
pp encodings
|
|
end
|
|
|
|
__END__
|
|
---
|
|
- actioncable-7.0.4.3
|
|
- actionmailbox-7.0.4.3
|
|
- actionmailer-7.0.4.3
|
|
- actionpack-7.0.4.3
|
|
- actiontext-7.0.4.3
|
|
- actionview-7.0.4.3
|
|
- activejob-7.0.4.3
|
|
- activemodel-7.0.4.3
|
|
- activerecord-7.0.4.3
|
|
- activestorage-7.0.4.3
|
|
- activesupport-7.0.4.3
|
|
- addressable-2.8.4
|
|
- autoprefixer-rails-10.4.13.0
|
|
- aws-partitions-1.744.0
|
|
- aws-sdk-cloudformation-1.77.0
|
|
- aws-sdk-cloudfront-1.76.0
|
|
- aws-sdk-cloudwatch-1.72.0
|
|
- aws-sdk-core-3.171.0
|
|
- aws-sdk-dynamodb-1.83.0
|
|
- aws-sdk-ec2-1.375.0
|
|
- aws-sdk-iam-1.77.0
|
|
- aws-sdk-kinesis-1.45.0
|
|
- aws-sdk-kms-1.63.0
|
|
- aws-sdk-lambda-1.93.0
|
|
- aws-sdk-rds-1.175.0
|
|
- aws-sdk-resources-3.162.0
|
|
- aws-sdk-s3-1.120.1
|
|
- aws-sdk-secretsmanager-1.73.0
|
|
- aws-sdk-sns-1.60.0
|
|
- aws-sdk-ssm-1.150.0
|
|
- backports-3.24.1
|
|
- brakeman-5.4.1
|
|
- bundler-2.4.11
|
|
- capybara-3.39.0
|
|
- concurrent-ruby-1.2.2
|
|
- connection_pool-2.4.0
|
|
- dalli-3.2.4
|
|
- database_cleaner-2.0.2
|
|
- devise-4.9.2
|
|
- dry-types-1.7.1
|
|
- elasticsearch-8.7.0
|
|
- elasticsearch-api-8.7.0
|
|
- excon-0.99.0
|
|
- faker-3.1.1
|
|
- faraday-retry-2.1.0
|
|
- fastlane-2.212.1
|
|
- fog-aws-3.18.0
|
|
- git-1.18.0
|
|
- google-cloud-errors-1.3.1
|
|
- google-protobuf-3.22.2
|
|
- googleauth-1.5.1
|
|
- graphql-2.0.21
|
|
- grpc-1.53.0
|
|
- jwt-2.7.0
|
|
- loofah-2.20.0
|
|
- mail-2.8.1
|
|
- mime-types-data-3.2023.0218.1
|
|
- minitest-5.18.0
|
|
- msgpack-1.7.0
|
|
- net-http-persistent-4.0.2
|
|
- net-ssh-7.1.0
|
|
- newrelic_rpm-9.1.0
|
|
- nio4r-2.5.9
|
|
- nokogiri-1.14.3
|
|
- octokit-6.1.1
|
|
- oj-3.14.3
|
|
- parser-3.2.2.0
|
|
- pg-1.4.6
|
|
- plist-3.7.0
|
|
- puma-6.2.1
|
|
- rack-3.0.7
|
|
- rack-cors-2.0.1
|
|
- rack-protection-3.0.6
|
|
- rack-test-2.1.0
|
|
- rails-7.0.4.3
|
|
- railties-7.0.4.3
|
|
- raindrops-0.20.1
|
|
- redis-store-1.9.2
|
|
- regexp_parser-2.7.0
|
|
- responders-3.1.0
|
|
- rouge-4.1.0
|
|
- rspec-core-3.12.1
|
|
- rspec-mocks-3.12.5
|
|
- rubocop-1.50.0
|
|
- rubocop-ast-1.28.0
|
|
- rubocop-performance-1.17.1
|
|
- rubocop-rails-2.19.0
|
|
- rubocop-rspec-2.19.0
|
|
- ruby-progressbar-1.13.0
|
|
- ruby_parser-3.20.0
|
|
- rubygems-update-3.4.11
|
|
- selenium-webdriver-4.8.6
|
|
- sidekiq-7.0.8
|
|
- sinatra-3.0.6
|
|
- slop-4.10.1
|
|
- sqlite3-1.6.2
|
|
- thin-1.8.2
|
|
- tilt-2.1.0
|
|
- yard-0.9.32
|
|
- zeitwerk-2.6.7
|