From c6264eb1bffa3b467c8aff460569b22f6dfa3dc5 Mon Sep 17 00:00:00 2001 From: Oliver Eyton-Williams Date: Thu, 21 Jan 2021 10:25:51 +0100 Subject: [PATCH 1/8] fix: remove cdn entirely and rely on Cloudflare --- assets/javascripts/app/config.coffee.erb | 2 +- lib/app.rb | 8 ++------ lib/tasks/docs.thor | 1 + public/opensearch.xml | 4 ++-- views/index.erb | 24 ++++++++++++------------ views/other.erb | 6 +++--- 6 files changed, 21 insertions(+), 24 deletions(-) diff --git a/assets/javascripts/app/config.coffee.erb b/assets/javascripts/app/config.coffee.erb index a822f7e2..97e91ace 100644 --- a/assets/javascripts/app/config.coffee.erb +++ b/assets/javascripts/app/config.coffee.erb @@ -12,7 +12,7 @@ app.config = sentry_dsn: '<%= App.sentry_dsn %>' version: <%= Time.now.to_i %> release: <%= Time.now.utc.httpdate.to_json %> - mathml_stylesheet: '<%= App.cdn_origin %>/mathml.css' + mathml_stylesheet: '/mathml.css' favicon_spritesheet: '<%= image_path('sprites/docs.png') %>' service_worker_path: '/service-worker.js' service_worker_enabled: <%= App.environment == :production || ENV['ENABLE_SERVICE_WORKER'] == 'true' %> diff --git a/lib/app.rb b/lib/app.rb index c1198f86..917d1a03 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -20,8 +20,6 @@ class App < Sinatra::Application set :root, Pathname.new(File.expand_path('../..', __FILE__)) set :sprockets, Sprockets::Environment.new(root) - set :cdn_origin, '' - set :assets_prefix, 'assets' set :assets_path, File.join(public_folder, assets_prefix) set :assets_manifest_path, File.join(assets_path, 'manifest.json') @@ -75,9 +73,8 @@ class App < Sinatra::Application configure :production do set :static, false - set :cdn_origin, 'https://cdn.devdocs.io' set :docs_origin, '//docs.devdocs.io' - set :csp, "default-src 'self' *; script-src 'self' 'nonce-devdocs' https://cdn.devdocs.io https://www.google-analytics.com https://secure.gaug.es https://*.jquery.com; font-src 'none'; style-src 'self' 'unsafe-inline' *; img-src 'self' * data:;" + set :csp, "default-src 'self' *; script-src 'self' 'nonce-devdocs' https://www.google-analytics.com https://secure.gaug.es https://*.jquery.com; font-src 'none'; style-src 'self' 'unsafe-inline' *; img-src 'self' * data:;" use Rack::ConditionalGet use Rack::ETag @@ -102,7 +99,6 @@ class App < Sinatra::Application Sprockets::Helpers.configure do |config| config.digest = true - config.asset_host = 'cdn.devdocs.io' config.manifest = Sprockets::Manifest.new(sprockets, assets_manifest_path) end end @@ -202,7 +198,7 @@ class App < Sinatra::Application def service_worker_asset_urls @@service_worker_asset_urls ||= [ - javascript_path('application', asset_host: false), + javascript_path('application'), stylesheet_path('application'), image_path('sprites/docs.png'), image_path('sprites/docs@2x.png'), diff --git a/lib/tasks/docs.thor b/lib/tasks/docs.thor index 5ed3f5cf..4c41a1be 100644 --- a/lib/tasks/docs.thor +++ b/lib/tasks/docs.thor @@ -198,6 +198,7 @@ class DocsCLI < Thor puts '[S3] Done syncing.' # Upload packages to dl.devdocs.io (used by the "thor docs:download" command) + # TODO(MIGRATION): replace this with an S3 bucket upload. puts '[MaxCDN] Begin uploading.' Net::SFTP.start('ftp.devdocs-dl.devdocs.netdna-cdn.com', ENV['DEVDOCS_DL_USERNAME'], password: ENV['DEVDOCS_DL_PASSWORD']) do |sftp| docs.each do |doc| diff --git a/public/opensearch.xml b/public/opensearch.xml index 734c4249..9c78ffdb 100644 --- a/public/opensearch.xml +++ b/public/opensearch.xml @@ -4,8 +4,8 @@ Search API documentation devdocs - https://cdn.devdocs.io/favicon.ico - https://cdn.devdocs.io/images/icon-64.png + https://devdocs.io/favicon.ico + https://devdocs.io/images/icon-64.png UTF-8 https://devdocs.io diff --git a/views/index.erb b/views/index.erb index 4fb96153..ce0cb887 100644 --- a/views/index.erb +++ b/views/index.erb @@ -9,7 +9,7 @@ - + @@ -21,22 +21,22 @@ - - - - - - - - - - + + + + + + + + + + <%= stylesheet_tag 'application' %> <%= erb :app -%> -<%= javascript_tag 'application', asset_host: false %> +<%= javascript_tag 'application' %> <%= javascript_tag 'docs' %><% unless App.production? %> <%= javascript_tag 'debug' %><% end %> diff --git a/views/other.erb b/views/other.erb index 45cab943..abd8bc20 100644 --- a/views/other.erb +++ b/views/other.erb @@ -6,18 +6,18 @@ <% if doc_index_page? %><% else %><% end %> - + DevDocs<%= " — #{@doc['full_name']} documentation" if doc_index_page? %> - + <%= stylesheet_tag 'application' %> <%= erb :app -%> -<%= javascript_tag 'application', asset_host: false %><% unless App.production? %> +<%= javascript_tag 'application' %><% unless App.production? %> <%= javascript_tag 'debug' %><% end %> From 7713074c938136afe6f9c284ae2057897471e636 Mon Sep 17 00:00:00 2001 From: Oliver Eyton-Williams Date: Fri, 22 Jan 2021 18:19:49 +0100 Subject: [PATCH 2/8] feat: use the new buckets The tarballs are currently getting put into /bundles, but maybe this isn't necessary --- lib/tasks/docs.thor | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/lib/tasks/docs.thor b/lib/tasks/docs.thor index 4c41a1be..202bf17a 100644 --- a/lib/tasks/docs.thor +++ b/lib/tasks/docs.thor @@ -191,7 +191,7 @@ class DocsCLI < Thor puts '[S3] Begin syncing.' docs.each do |doc| puts "[S3] Syncing #{doc.path}..." - cmd = "aws s3 sync #{File.join(Docs.store_path, doc.path)} s3://devdocs-assets/#{doc.path} --delete --profile devdocs" + cmd = "aws s3 sync #{File.join(Docs.store_path, doc.path)} s3://devdocs-staging-documents/#{doc.path} --delete --profile devdocs" cmd << ' --dryrun' if options[:dryrun] system(cmd) end @@ -199,20 +199,16 @@ class DocsCLI < Thor # Upload packages to dl.devdocs.io (used by the "thor docs:download" command) # TODO(MIGRATION): replace this with an S3 bucket upload. - puts '[MaxCDN] Begin uploading.' - Net::SFTP.start('ftp.devdocs-dl.devdocs.netdna-cdn.com', ENV['DEVDOCS_DL_USERNAME'], password: ENV['DEVDOCS_DL_PASSWORD']) do |sftp| - docs.each do |doc| - filename = "#{doc.path}.tar.gz" - print "[MaxCDN] Uploading #{filename}..." - if options[:dryrun] - print "\n" - else - sftp.upload! File.join(Docs.store_path, filename), File.join('', 'public_html', filename) - print " OK\n" - end - end + puts '[S3 bundle] Begin uploading.' + + docs.each do |doc| + filename = "#{doc.path}.tar.gz" + print "[S3 bundle] Uploading #{filename}..." + cmd = "aws s3 cp #{File.join(Docs.store_path, filename)} s3://devdocs-staging-downloads/bundles/#{filename} --profile devdocs" + cmd << ' --dryrun' if options[:dryrun] + system(cmd) end - puts '[MaxCDN] Done uploading.' + puts '[S3 bundle] Done uploading.' end desc 'commit', '[private]' @@ -245,7 +241,7 @@ class DocsCLI < Thor FileUtils.mkpath(dir) ['index.json', 'meta.json'].each do |filename| - json = "https://docs.devdocs.io/#{doc.path}/#{filename}?#{time}" + json = "https://documents.devdocs.in/#{doc.path}/#{filename}?#{time}" begin open(json) do |file| mutex.synchronize do @@ -342,7 +338,7 @@ class DocsCLI < Thor def download_doc(doc) target_path = File.join(Docs.store_path, doc.path) - open "http://dl.devdocs.io/#{doc.path}.tar.gz" do |file| + open "https://downloads.devdocs.in/bundles/#{doc.path}.tar.gz" do |file| FileUtils.mkpath(target_path) file.close tar = UnixUtils.gunzip(file.path) From f8aaaf4f61f5acb6c39f22d4b8f2a4da86f02b6d Mon Sep 17 00:00:00 2001 From: Oliver Eyton-Williams Date: Fri, 22 Jan 2021 20:00:17 +0100 Subject: [PATCH 3/8] fix: change docs_origin to documents.devdocs.in --- lib/app.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/app.rb b/lib/app.rb index 917d1a03..18141eee 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -73,7 +73,7 @@ class App < Sinatra::Application configure :production do set :static, false - set :docs_origin, '//docs.devdocs.io' + set :docs_origin, '//documents.devdocs.in' set :csp, "default-src 'self' *; script-src 'self' 'nonce-devdocs' https://www.google-analytics.com https://secure.gaug.es https://*.jquery.com; font-src 'none'; style-src 'self' 'unsafe-inline' *; img-src 'self' * data:;" use Rack::ConditionalGet From 72ac0a2101c603e0b7d9ee498d1245fb0ab71f31 Mon Sep 17 00:00:00 2001 From: Oliver Eyton-Williams Date: Tue, 16 Feb 2021 11:49:32 +0100 Subject: [PATCH 4/8] fix: s/devdocs.in/devdocs.io/g --- lib/app.rb | 2 +- lib/tasks/docs.thor | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/app.rb b/lib/app.rb index 18141eee..15a9a8ec 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -73,7 +73,7 @@ class App < Sinatra::Application configure :production do set :static, false - set :docs_origin, '//documents.devdocs.in' + set :docs_origin, '//documents.devdocs.io' set :csp, "default-src 'self' *; script-src 'self' 'nonce-devdocs' https://www.google-analytics.com https://secure.gaug.es https://*.jquery.com; font-src 'none'; style-src 'self' 'unsafe-inline' *; img-src 'self' * data:;" use Rack::ConditionalGet diff --git a/lib/tasks/docs.thor b/lib/tasks/docs.thor index 202bf17a..bd7bf050 100644 --- a/lib/tasks/docs.thor +++ b/lib/tasks/docs.thor @@ -241,7 +241,7 @@ class DocsCLI < Thor FileUtils.mkpath(dir) ['index.json', 'meta.json'].each do |filename| - json = "https://documents.devdocs.in/#{doc.path}/#{filename}?#{time}" + json = "https://documents.devdocs.io/#{doc.path}/#{filename}?#{time}" begin open(json) do |file| mutex.synchronize do @@ -338,7 +338,7 @@ class DocsCLI < Thor def download_doc(doc) target_path = File.join(Docs.store_path, doc.path) - open "https://downloads.devdocs.in/bundles/#{doc.path}.tar.gz" do |file| + open "https://downloads.devdocs.io/bundles/#{doc.path}.tar.gz" do |file| FileUtils.mkpath(target_path) file.close tar = UnixUtils.gunzip(file.path) From 60722327eefcd64bac3079e53bf6be26dd8e2bb8 Mon Sep 17 00:00:00 2001 From: Oliver Eyton-Williams Date: Tue, 16 Feb 2021 12:00:13 +0100 Subject: [PATCH 5/8] fix: update staging bucket references to prod --- lib/tasks/docs.thor | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tasks/docs.thor b/lib/tasks/docs.thor index bd7bf050..4ac8d542 100644 --- a/lib/tasks/docs.thor +++ b/lib/tasks/docs.thor @@ -191,7 +191,7 @@ class DocsCLI < Thor puts '[S3] Begin syncing.' docs.each do |doc| puts "[S3] Syncing #{doc.path}..." - cmd = "aws s3 sync #{File.join(Docs.store_path, doc.path)} s3://devdocs-staging-documents/#{doc.path} --delete --profile devdocs" + cmd = "aws s3 sync #{File.join(Docs.store_path, doc.path)} s3://devdocs-documents/#{doc.path} --delete --profile devdocs" cmd << ' --dryrun' if options[:dryrun] system(cmd) end @@ -204,7 +204,7 @@ class DocsCLI < Thor docs.each do |doc| filename = "#{doc.path}.tar.gz" print "[S3 bundle] Uploading #{filename}..." - cmd = "aws s3 cp #{File.join(Docs.store_path, filename)} s3://devdocs-staging-downloads/bundles/#{filename} --profile devdocs" + cmd = "aws s3 cp #{File.join(Docs.store_path, filename)} s3://devdocs-downloads/bundles/#{filename} --profile devdocs" cmd << ' --dryrun' if options[:dryrun] system(cmd) end From 23b84c109e82303cc04ad81463b7b1f81c2e9f1b Mon Sep 17 00:00:00 2001 From: Oliver Eyton-Williams Date: Tue, 16 Feb 2021 12:04:35 +0100 Subject: [PATCH 6/8] fix: update comments --- lib/tasks/docs.thor | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/tasks/docs.thor b/lib/tasks/docs.thor index 4ac8d542..ec4f8a70 100644 --- a/lib/tasks/docs.thor +++ b/lib/tasks/docs.thor @@ -197,8 +197,7 @@ class DocsCLI < Thor end puts '[S3] Done syncing.' - # Upload packages to dl.devdocs.io (used by the "thor docs:download" command) - # TODO(MIGRATION): replace this with an S3 bucket upload. + # Upload packages to downloads.devdocs.io (used by the "thor docs:download" command) puts '[S3 bundle] Begin uploading.' docs.each do |doc| From 32c94604a6d2c774573a504bb15ab769c76c830b Mon Sep 17 00:00:00 2001 From: Oliver Eyton-Williams Date: Tue, 16 Feb 2021 12:06:09 +0100 Subject: [PATCH 7/8] fix: remove the 'bundle' path segment --- lib/tasks/docs.thor | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tasks/docs.thor b/lib/tasks/docs.thor index ec4f8a70..d7f93f46 100644 --- a/lib/tasks/docs.thor +++ b/lib/tasks/docs.thor @@ -203,7 +203,7 @@ class DocsCLI < Thor docs.each do |doc| filename = "#{doc.path}.tar.gz" print "[S3 bundle] Uploading #{filename}..." - cmd = "aws s3 cp #{File.join(Docs.store_path, filename)} s3://devdocs-downloads/bundles/#{filename} --profile devdocs" + cmd = "aws s3 cp #{File.join(Docs.store_path, filename)} s3://devdocs-downloads/#{filename} --profile devdocs" cmd << ' --dryrun' if options[:dryrun] system(cmd) end @@ -337,7 +337,7 @@ class DocsCLI < Thor def download_doc(doc) target_path = File.join(Docs.store_path, doc.path) - open "https://downloads.devdocs.io/bundles/#{doc.path}.tar.gz" do |file| + open "https://downloads.devdocs.io/#{doc.path}.tar.gz" do |file| FileUtils.mkpath(target_path) file.close tar = UnixUtils.gunzip(file.path) From 8eee6128529ed91c641098f664c5ebfcb1663753 Mon Sep 17 00:00:00 2001 From: Oliver Eyton-Williams Date: Tue, 16 Feb 2021 17:25:55 +0100 Subject: [PATCH 8/8] fix: puts, not print Co-authored-by: Simon Legner --- lib/tasks/docs.thor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/docs.thor b/lib/tasks/docs.thor index d7f93f46..0ecd7bdb 100644 --- a/lib/tasks/docs.thor +++ b/lib/tasks/docs.thor @@ -202,7 +202,7 @@ class DocsCLI < Thor docs.each do |doc| filename = "#{doc.path}.tar.gz" - print "[S3 bundle] Uploading #{filename}..." + puts "[S3 bundle] Uploading #{filename}..." cmd = "aws s3 cp #{File.join(Docs.store_path, filename)} s3://devdocs-downloads/#{filename} --profile devdocs" cmd << ' --dryrun' if options[:dryrun] system(cmd)