Merge pull request #939 from JuliaDocs/backports-0.21.2

Backports for 0.21.2

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Documenter.jl changelog
 
+## Version `v0.21.2`
+
+* ![Bugfix][badge-bugfix] `linkcheck` now handles servers that do not support `HEAD` requests
+ and properly checks for status codes of FTP responses. ([#934][github-934])
+
 ## Version `v0.21.1`
 
 * ![Bugfix][badge-bugfix] `@repl` blocks now work correctly together with quoted
@@ -16,15 +21,15 @@
 
  **For upgrading:** If you don't specify `format` (i.e. you rely on the default) you don't have to do anything.
  Otherwise update calls to `makedocs` to use struct instances instead of symbols, e.g.
- 
+
  ```
  makedocs(
  format = :markdown
  )
  ```
- 
+
  should be changed to
- 
+
  ```
  using DocumenterMarkdown
  makedocs(
@@ -51,7 +56,7 @@
  ...
  )
  ```
- 
+
  _**Note:** It is technically possible to specify the same argument twice with different values by passing both variants. In that case the value to the deprecated `html_*` variant takes precedence._
 
 * ![Feature][badge-feature] Packages extending Documenter can now define subtypes of `Documenter.Plugin`,

src/DocChecks.jl

@@ -169,7 +169,7 @@ function linkcheck(doc::Documents.Document)
  return nothing
 end
 
-function linkcheck(link::Markdown.Link, doc::Documents.Document)
+function linkcheck(link::Markdown.Link, doc::Documents.Document; method::Symbol=:HEAD)
 
  # first, make sure we're not supposed to ignore this link
  for r in doc.user.linkcheck_ignore
@@ -180,7 +180,8 @@ function linkcheck(link::Markdown.Link, doc::Documents.Document)
  end
 
  if !haskey(doc.internal.locallinks, link)
- cmd = `curl -sI --proto =http,https,ftp,ftps $(link.url) --max-time 10`
+ null_file = @static Sys.iswindows() ? "nul" : "/dev/null"
+ cmd = `curl $(method === :HEAD ? "-sI" : "-s") --proto =http,https,ftp,ftps $(link.url) --max-time 10 -o $null_file --write-out "%{http_code} %{url_effective} %{redirect_url}"`
 
  local result
  try
@@ -191,27 +192,32 @@ function linkcheck(link::Markdown.Link, doc::Documents.Document)
  @warn "$cmd failed:" exception = err
  return false
  end
- HTTP_STATUS_REGEX = r"^HTTP/(1.1|2) (\d+) (.+)$"m
- FTP_STATUS_REGEX = r"^Last-Modified: (.+)\r\nContent-Length: (\d+)(?:\r\n(.*))?$"s
- if occursin(HTTP_STATUS_REGEX, result)
- status = parse(Int, match(HTTP_STATUS_REGEX, result).captures[2])
- if status < 300
+ STATUS_REGEX = r"^(\d+) (\w+)://(?:\S+) (\S+)?$"m
+ matched = match(STATUS_REGEX, result)
+ if matched !== nothing
+ status, scheme, location = matched.captures
+ status = parse(Int, status)
+ scheme = uppercase(scheme)
+ protocol = startswith(scheme, "HTTP") ? :HTTP :
+ startswith(scheme, "FTP") ? :FTP : :UNKNOWN
+
+ if (protocol === :HTTP && status < 300) ||
+ (protocol === :FTP && (200 <= status < 300 || status == 350))
  @debug "linkcheck '$(link.url)' status: $(status)."
- elseif status < 400
- LOCATION_REGEX = r"^Location: (.+)$"m
- if occursin(LOCATION_REGEX, result)
- location = strip(match(LOCATION_REGEX, result).captures[1])
+ elseif protocol === :HTTP && status < 400
+ if location !== nothing
  @warn "linkcheck '$(link.url)' status: $(status), redirects to $(location)."
  else
  @warn "linkcheck '$(link.url)' status: $(status)."
  end
+ elseif protocol === :HTTP && status == 405 && method === :HEAD
+ # when a server doesn't support HEAD requests, fallback to GET
+ @debug "linkcheck '$(link.url)' status: $(status), retrying without `-I`"
+ return linkcheck(link, doc; method=:GET)
  else
  push!(doc.internal.errors, :linkcheck)
  @error "linkcheck '$(link.url)' status: $(status)."
  end
- elseif occursin(FTP_STATUS_REGEX, result)
- # this regex is matched iff success
- @debug "linkcheck '$(link.url)': FTP success"
  else
  push!(doc.internal.errors, :linkcheck)
  @warn "invalid result returned by $cmd:" result

test/docchecks.jl

@@ -14,6 +14,7 @@ using Documenter.Documents
  [FTP success](ftp://ftp.iana.org/tz/data/etcetera)
  [FTP (no proto) success](ftp.iana.org/tz/data/etcetera)
  [Redirect success](google.com)
+ [HEAD fail GET success](https://codecov.io/gh/invenia/LibPQ.jl)
  """
 
  Documents.walk(Dict{Symbol, Any}(), src) do block