crictl pull image from private registries return 500 error message

[gtn-tathagata]

Unable to locate the problem, seek help~

1. command: crictl pull harbor.nebula.com/base/df/nginx:0911

2. return error:

E1016 16:43:09.501206 2644669 remote_image.go:180] "PullImage from image service failed" err="rpc error: code = Unknown desc = failed to pull and unpack image "harbor.nebula.com/base/df/nginx:0911": failed to resolve reference "harbor.nebula.com/base/df/nginx:0911": pulling from host 127.0.0.1:65001 failed with status code [manifests 0911]: 500 Internal Server Error" image="harbor.nebula.com/base/df/nginx:0911"
FATA[0000] pulling image: failed to pull and unpack image "harbor.nebula.com/base/df/nginx:0911": failed to resolve reference "harbor.nebula.com/base/df/nginx:0911": pulling from host 127.0.0.1:65001 failed with status code [manifests 0911]: 500 Internal Server Error

3. dfdaemon pod core.log:

{"level":"info","ts":"2024-10-16 08:38:18.037","caller":"cmd/daemon.go:148","msg":"Version:\nMajor: 2, Minor: 0, GitVersion: v2.1.0, GitCommit: e49b0af, Platform: linux, BuildTime: 2024-04-10T11:46:42Z, GoVersion: go1.21.1 linux/amd64, Gotags: none, Gogcflags: none"}
{"level":"info","ts":"2024-10-16 08:38:18.039","caller":"cmd/daemon.go:194","msg":"daemon is launched by pid: -1"}
{"level":"info","ts":"2024-10-16 08:38:18.048","caller":"client/client_v1.go:78","msg":"manager address dns:///dragonfly-manager.dragonfly-system.svc.cluster.local:65003 is reachable"}
{"level":"info","ts":"2024-10-16 08:38:18.064","caller":"peer/piece_manager.go:100","msg":"set download limiter 1073741824.000000 for piece manager"}
{"level":"info","ts":"2024-10-16 08:38:18.064","caller":"peer/piece_manager.go:92","msg":"set calculateDigest to true for piece manager"}
{"level":"warn","ts":"2024-10-16 08:38:18.064","caller":"peer/traffic_shaper.go:59","msg":"type "" doesn't exist, use plain traffic shaper instead","stacktrace":"d7y.io/dragonfly/v2/client/daemon/peer.NewTrafficShaper\n\t/go/src/d7y.io/dragonfly/v2/client/daemon/peer/traffic_shaper.go:59\nd7y.io/dragonfly/v2/client/daemon/peer.NewPeerTaskManager\n\t/go/src/d7y.io/dragonfly/v2/client/daemon/peer/peertask_manager.go:151\nd7y.io/dragonfly/v2/client/daemon.New\n\t/go/src/d7y.io/dragonfly/v2/client/daemon/daemon.go:338\nd7y.io/dragonfly/v2/cmd/dfget/cmd.runDaemon\n\t/go/src/d7y.io/dragonfly/v2/cmd/dfget/cmd/daemon.go:199\nd7y.io/dragonfly/v2/cmd/dfget/cmd.glob..func1\n\t/go/src/d7y.io/dragonfly/v2/cmd/dfget/cmd/daemon.go:85\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:940\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:1068\ngithub.com/spf13/cobra.(Command).Execute\n\t/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:992\nd7y.io/dragonfly/v2/cmd/dfget/cmd.Execute\n\t/go/src/d7y.io/dragonfly/v2/cmd/dfget/cmd/root.go:125\nmain.main\n\t/go/src/d7y.io/dragonfly/v2/cmd/dfget/main.go:24\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:267"}
{"level":"info","ts":"2024-10-16 08:38:18.064","caller":"proxy/proxy_manager.go:90","msg":"registry mirror: https://harbor.nebula.com"}
{"level":"info","ts":"2024-10-16 08:38:18.064","caller":"proxy/proxy_manager.go:99","msg":"load 1 proxy rules"}
{"level":"info","ts":"2024-10-16 08:38:18.064","caller":"proxy/proxy_manager.go:109","msg":"[1] proxy blobs/sha256. with dragonfly "}
{"level":"info","ts":"2024-10-16 08:38:18.065","caller":"daemon/daemon.go:645","msg":"serve download grpc at unix://"}
{"level":"info","ts":"2024-10-16 08:38:18.065","caller":"daemon/daemon_linux.go:43","msg":"switch net namespace, from /proc/2587653/ns/net to /run/dragonfly/net"}
{"level":"info","ts":"2024-10-16 08:38:18.065","caller":"daemon/daemon_linux.go:55","msg":"recover net namespace, from /run/dragonfly/net to /proc/2587653/ns/net"}
{"level":"info","ts":"2024-10-16 08:38:18.065","caller":"daemon/daemon.go:656","msg":"serve peer grpc at tcp://[::]:65000"}
{"level":"info","ts":"2024-10-16 08:38:18.065","caller":"daemon/daemon_linux.go:60","msg":"recover net namespace, close original fd"}
{"level":"info","ts":"2024-10-16 08:38:18.066","caller":"daemon/daemon.go:741","msg":"serve upload service at tcp://[::]:65002"}
{"level":"info","ts":"2024-10-16 08:38:18.066","caller":"daemon/daemon.go:779","msg":"serve announcer"}
{"level":"info","ts":"2024-10-16 08:38:18.066","caller":"announcer/announcer.go:105","msg":"announce seed peer to manager"}
{"level":"info","ts":"2024-10-16 08:38:18.066","caller":"announcer/announcer.go:111","msg":"announce peer to scheduler"}
{"level":"info","ts":"2024-10-16 08:38:18.066","caller":"daemon/daemon.go:681","msg":"serve proxy at tcp://0.0.0.0:65001"}
{"level":"info","ts":"2024-10-16 08:38:18.066","caller":"daemon/daemon.go:872","msg":"serve http health at &config.TCPListenOption{Listen:"0.0.0.0", PortRange:config.TCPListenPortRange{Start:40901, End:0}, Namespace:""}"}
{"level":"info","ts":"2024-10-16 08:39:18.066","caller":"storage/storage_manager.go:985","msg":"marked 0 task(s), reclaimed 0 task(s)"}
{"level":"info","ts":"2024-10-16 08:40:18.066","caller":"storage/storage_manager.go:985","msg":"marked 0 task(s), reclaimed 0 task(s)"}
{"level":"info","ts":"2024-10-16 08:41:18.065","caller":"storage/storage_manager.go:985","msg":"marked 0 task(s), reclaimed 0 task(s)"}
{"level":"error","ts":"2024-10-16 08:42:01.351","caller":"transport/transport.go:222","msg":"round trip error: dial tcp: lookup harbor.nebula.com on 10.244.64.10:53: no such host","method":"HEAD","url":"https://harbor.nebula.com/v2/base/df/nginx/manifests/0911?ns=harbor.nebula.com","stacktrace":"d7y.io/dragonfly/v2/client/daemon/transport.(*transport).RoundTrip\n\t/go/src/d7y.io/dragonfly/v2/client/daemon/transport/transport.go:222\nnet/http/httputil.(*ReverseProxy).ServeHTTP\n\t/usr/local/go/src/net/http/httputil/reverseproxy.go:473\nd7y.io/dragonfly/v2/client/daemon/proxy.(*Proxy).mirrorRegistry\n\t/go/src/d7y.io/dragonfly/v2/client/daemon/proxy/proxy.go:575\nnet/http.HandlerFunc.ServeHTTP\n\t/usr/local/go/src/net/http/server.go:2136\nnet/http.(*ServeMux).ServeHTTP\n\t/usr/local/go/src/net/http/server.go:2514\nd7y.io/dragonfly/v2/client/daemon/proxy.(*Proxy).ServeHTTP\n\t/go/src/d7y.io/dragonfly/v2/client/daemon/proxy/proxy.go:372\nnet/http.serverHandler.ServeHTTP\n\t/usr/local/go/src/net/http/server.go:2938\nnet/http.(*conn).serve\n\t/usr/local/go/src/net/http/server.go:2009"}
{"level":"info","ts":"2024-10-16 08:42:18.066","caller":"storage/storage_manager.go:985","msg":"marked 0 task(s), reclaimed 0 task(s)"}
{"level":"error","ts":"2024-10-16 08:43:09.481","caller":"transport/transport.go:222","msg":"round trip error: dial tcp: lookup harbor.nebula.com on 10.244.64.10:53: no such host","method":"HEAD","url":"https://harbor.nebula.com/v2/base/df/nginx/manifests/0911?ns=harbor.nebula.com","stacktrace":"d7y.io/dragonfly/v2/client/daemon/transport.(*transport).RoundTrip\n\t/go/src/d7y.io/dragonfly/v2/client/daemon/transport/transport.go:222\nnet/http/httputil.(*ReverseProxy).ServeHTTP\n\t/usr/local/go/src/net/http/httputil/reverseproxy.go:473\nd7y.io/dragonfly/v2/client/daemon/proxy.(*Proxy).mirrorRegistry\n\t/go/src/d7y.io/dragonfly/v2/client/daemon/proxy/proxy.go:575\nnet/http.HandlerFunc.ServeHTTP\n\t/usr/local/go/src/net/http/server.go:2136\nnet/http.(*ServeMux).ServeHTTP\n\t/usr/local/go/src/net/http/server.go:2514\nd7y.io/dragonfly/v2/client/daemon/proxy.(*Proxy).ServeHTTP\n\t/go/src/d7y.io/dragonfly/v2/client/daemon/proxy/proxy.go:372\nnet/http.serverHandler.ServeHTTP\n\t/usr/local/go/src/net/http/server.go:2938\nnet/http.(*conn).serve\n\t/usr/local/go/src/net/http/server.go:2009"}

4.private registries

harbor1.10.18 self signed certificate

5.dragonfly-1.1.45 install with helm chars

values.yaml Modified parts：

containerRuntime:
containerd:
enable: true
injectConfigPath: true
registries:
- 'https://docker.io'
- "https://harbor.nebula.com"
dfdaemon:
registryMirror:
# -- When enabled, use value of "X-Dragonfly-Registry" in http header for remote instead of url host.
dynamic: true
# -- URL for the registry mirror.
url: https://index.docker.io
url: https://harbor.nebula.com

6.containerd config

/etc/containerd/config.toml

version = 2
root = "/var/lib/containerd"
state = "/run/containerd"
oom_score = -999

[grpc]
max_recv_message_size = 16777216
max_send_message_size = 16777216

[debug]
level = "info"

[metrics]
address = ""
grpc_histogram = false

[plugins]
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "registry.aliyuncs.com/kubeadm-ha/pause:3.9"
max_container_log_line_size = -1
[plugins."io.containerd.grpc.v1.cri".containerd]
default_runtime_name = "runc"
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
runtime_engine = ""
runtime_root = ""
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"

7./etc/containerd/certs.d

ls certs.d/harbor.nebula.com/
ca.crt hosts.toml nebula.com.cert nebula.com.key

cat harbor.nebula.com/hosts.toml
server = "https://harbor.nebula.com"
[host."http://127.0.0.1:65001"]
capabilities = ["pull", "resolve"]
[host."http://127.0.0.1:65001".header]
X-Dragonfly-Registry = ["https://harbor.nebula.com"]
[host."https://harbor.nebula.com"]
capabilities = ["pull", "resolve"]

8. dfdaemon dfget.yaml

aliveTime: 0s
gcInterval: 1m0s
keepStorage: false
workHome:
logDir:
cacheDir:
pluginDir:
dataDir: /var/lib/dragonfly
console: false
health:
path: /server/ping
tcpListen:
port: 40901
verbose: false
scheduler:
manager:
enable: true
netAddrs:
- type: tcp
addr: dragonfly-manager.dragonfly-system.svc.cluster.local:65003
refreshInterval: 10m
netAddrs:
scheduleTimeout: 30s
disableAutoBackSource: false
seedPeer:
clusterID: 1
enable: false
type: super
host:
idc: ""
location: ""
download:
calculateDigest: true
downloadGRPC:
security:
insecure: true
tlsVerify: true
unixListen:
socket: ""
peerGRPC:
security:
insecure: true
tcpListen:
port: 65000
perPeerRateLimit: 512Mi
prefetch: false
totalRateLimit: 1024Mi
upload:
rateLimit: 1024Mi
security:
insecure: true
tlsVerify: false
tcpListen:
port: 65002
objectStorage:
enable: false
filter: Expires&Signature&ns
maxReplicas: 3
security:
insecure: true
tlsVerify: true
tcpListen:
port: 65004
storage:
diskGCThreshold: 50Gi
multiplex: true
strategy: io.d7y.storage.v2.simple
taskExpireTime: 6h
proxy:
defaultFilter: Expires&Signature&ns
defaultTag:
tcpListen:
namespace: /run/dragonfly/net
port: 65001
security:
insecure: true
tlsVerify: false
registryMirror:
dynamic: true
insecure: true
url: https://harbor.nebula.com
proxies:
- regx: blobs/sha256.*
security:
autoIssueCert: false
caCert: ""
certSpec:
dnsNames: null
ipAddresses: null
validityPeriod: 4320h
tlsPolicy: prefer
tlsVerify: false
network:
enableIPv6: false
announcer:
schedulerInterval: 30s
networkTopology:
enable: false
probe:
interval: 20m

9. crictl version

Version: 0.1.0
RuntimeName: containerd
RuntimeVersion: 1.6.20
RuntimeApiVersion: v1