From 7466ecab31f4ac9a34be3bde35ec81a2a0bd72fc Mon Sep 17 00:00:00 2001 From: chris Date: Tue, 12 Jul 2022 05:43:15 +0200 Subject: [PATCH] Add harbor files to containers. --- harbor/LICENSE | 201 +++++++++++++ harbor/common.sh | 130 ++++++++ harbor/common/config/core/app.conf | 6 + harbor/common/config/core/env | 50 ++++ harbor/common/config/db/env | 1 + harbor/common/config/jobservice/config.yml | 36 +++ harbor/common/config/jobservice/env | 15 + harbor/common/config/log/logrotate.conf | 8 + harbor/common/config/log/rsyslog_docker.conf | 7 + harbor/common/config/nginx/nginx.conf | 130 ++++++++ harbor/common/config/nginx/nginx.conf~ | 130 ++++++++ harbor/common/config/portal/nginx.conf | 38 +++ harbor/common/config/registry/config.yml | 43 +++ harbor/common/config/registry/passwd | 1 + harbor/common/config/registry/root.crt | 0 harbor/common/config/registryctl/config.yml | 5 + harbor/common/config/registryctl/env | 2 + harbor/common/config/trivy-adapter/env | 19 ++ harbor/docker-compose.yml | 299 +++++++++++++++++++ harbor/harbor.yml | 247 +++++++++++++++ harbor/harbor.yml.tmpl | 247 +++++++++++++++ harbor/install.sh | 101 +++++++ harbor/prepare | 64 ++++ 23 files changed, 1780 insertions(+) create mode 100644 harbor/LICENSE create mode 100644 harbor/common.sh create mode 100644 harbor/common/config/core/app.conf create mode 100644 harbor/common/config/core/env create mode 100644 harbor/common/config/db/env create mode 100644 harbor/common/config/jobservice/config.yml create mode 100644 harbor/common/config/jobservice/env create mode 100644 harbor/common/config/log/logrotate.conf create mode 100644 harbor/common/config/log/rsyslog_docker.conf create mode 100644 harbor/common/config/nginx/nginx.conf create mode 100644 harbor/common/config/nginx/nginx.conf~ create mode 100644 harbor/common/config/portal/nginx.conf create mode 100644 harbor/common/config/registry/config.yml create mode 100644 harbor/common/config/registry/passwd create mode 100755 harbor/common/config/registry/root.crt create mode 100644 harbor/common/config/registryctl/config.yml create mode 100644 harbor/common/config/registryctl/env create mode 100644 harbor/common/config/trivy-adapter/env create mode 100644 harbor/docker-compose.yml create mode 100644 harbor/harbor.yml create mode 100644 harbor/harbor.yml.tmpl create mode 100755 harbor/install.sh create mode 100755 harbor/prepare diff --git a/harbor/LICENSE b/harbor/LICENSE new file mode 100644 index 0000000..4b9cffe --- /dev/null +++ b/harbor/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright Project Harbor Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/harbor/common.sh b/harbor/common.sh new file mode 100644 index 0000000..0db96f0 --- /dev/null +++ b/harbor/common.sh @@ -0,0 +1,130 @@ +#!/bin/bash +#docker version: 17.06.0+ +#docker-compose version: 1.18.0+ +#golang version: 1.12.0+ + +set +e +set -o noglob + +# +# Set Colors +# + +bold=$(tput bold) +underline=$(tput sgr 0 1) +reset=$(tput sgr0) + +red=$(tput setaf 1) +green=$(tput setaf 76) +white=$(tput setaf 7) +tan=$(tput setaf 202) +blue=$(tput setaf 25) + +# +# Headers and Logging +# + +underline() { printf "${underline}${bold}%s${reset}\n" "$@" +} +h1() { printf "\n${underline}${bold}${blue}%s${reset}\n" "$@" +} +h2() { printf "\n${underline}${bold}${white}%s${reset}\n" "$@" +} +debug() { printf "${white}%s${reset}\n" "$@" +} +info() { printf "${white}➜ %s${reset}\n" "$@" +} +success() { printf "${green}✔ %s${reset}\n" "$@" +} +error() { printf "${red}✖ %s${reset}\n" "$@" +} +warn() { printf "${tan}➜ %s${reset}\n" "$@" +} +bold() { printf "${bold}%s${reset}\n" "$@" +} +note() { printf "\n${underline}${bold}${blue}Note:${reset} ${blue}%s${reset}\n" "$@" +} + +set -e + +function check_golang { + if ! go version &> /dev/null + then + warn "No golang package in your enviroment. You should use golang docker image build binary." + return + fi + + # docker has been installed and check its version + if [[ $(go version) =~ (([0-9]+)\.([0-9]+)([\.0-9]*)) ]] + then + golang_version=${BASH_REMATCH[1]} + golang_version_part1=${BASH_REMATCH[2]} + golang_version_part2=${BASH_REMATCH[3]} + + # the version of golang does not meet the requirement + if [ "$golang_version_part1" -lt 1 ] || ([ "$golang_version_part1" -eq 1 ] && [ "$golang_version_part2" -lt 12 ]) + then + warn "Better to upgrade golang package to 1.12.0+ or use golang docker image build binary." + return + else + note "golang version: $golang_version" + fi + else + warn "Failed to parse golang version." + return + fi +} + +function check_docker { + if ! docker --version &> /dev/null + then + error "Need to install docker(17.06.0+) first and run this script again." + exit 1 + fi + + # docker has been installed and check its version + if [[ $(docker --version) =~ (([0-9]+)\.([0-9]+)([\.0-9]*)) ]] + then + docker_version=${BASH_REMATCH[1]} + docker_version_part1=${BASH_REMATCH[2]} + docker_version_part2=${BASH_REMATCH[3]} + + note "docker version: $docker_version" + # the version of docker does not meet the requirement + if [ "$docker_version_part1" -lt 17 ] || ([ "$docker_version_part1" -eq 17 ] && [ "$docker_version_part2" -lt 6 ]) + then + error "Need to upgrade docker package to 17.06.0+." + exit 1 + fi + else + error "Failed to parse docker version." + exit 1 + fi +} + +function check_dockercompose { + if ! docker compose version &> /dev/null + then + error "Need to install docker compose(1.18.0+) by yourself first and run this script again." + exit 1 + fi + + # docker compose has been installed, check its version + if [[ $(docker compose version) =~ (([0-9]+)\.([0-9]+)([\.0-9]*)) ]] + then + docker_compose_version=${BASH_REMATCH[1]} + docker_compose_version_part1=${BASH_REMATCH[2]} + docker_compose_version_part2=${BASH_REMATCH[3]} + + note "docker compose version: $docker_compose_version" + # the version of docker compose does not meet the requirement + if [ "$docker_compose_version_part1" -lt 1 ] || ([ "$docker_compose_version_part1" -eq 1 ] && [ "$docker_compose_version_part2" -lt 18 ]) + then + error "Need to upgrade docker compose package to 1.18.0+." + exit 1 + fi + else + error "Failed to parse docker compose version." + exit 1 + fi +} diff --git a/harbor/common/config/core/app.conf b/harbor/common/config/core/app.conf new file mode 100644 index 0000000..28351cd --- /dev/null +++ b/harbor/common/config/core/app.conf @@ -0,0 +1,6 @@ +appname = Harbor +runmode = prod +enablegzip = true + +[prod] +httpport = 8080 diff --git a/harbor/common/config/core/env b/harbor/common/config/core/env new file mode 100644 index 0000000..e7a14b4 --- /dev/null +++ b/harbor/common/config/core/env @@ -0,0 +1,50 @@ +CONFIG_PATH=/etc/core/app.conf +UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem +_REDIS_URL_CORE=redis://redis:6379?idle_timeout_seconds=30 +SYNC_QUOTA=true +CHART_CACHE_DRIVER=redis +_REDIS_URL_REG=redis://redis:6379/1?idle_timeout_seconds=30 + +LOG_LEVEL=info +EXT_ENDPOINT=https://cs-registry.ddnss.de +DATABASE_TYPE=postgresql +POSTGRESQL_HOST=postgresql +POSTGRESQL_PORT=5432 +POSTGRESQL_USERNAME=postgres +POSTGRESQL_PASSWORD=mH3^tE6FPhUxk#W5iBP9FTq^AsyB8g +POSTGRESQL_DATABASE=registry +POSTGRESQL_SSLMODE=disable +POSTGRESQL_MAX_IDLE_CONNS=100 +POSTGRESQL_MAX_OPEN_CONNS=900 +REGISTRY_URL=http://registry:5000 +PORTAL_URL=http://portal:8080 +TOKEN_SERVICE_URL=http://core:8080/service/token +HARBOR_ADMIN_PASSWORD=j2Q2gRX@zpGYGsUZwJ@ynvnU3gw6Y* +MAX_JOB_WORKERS=10 +CORE_SECRET=Q1kOSJ2hbw3qs2Uh +JOBSERVICE_SECRET=LURd3ymSGca6nuB5 +WITH_NOTARY=False +WITH_TRIVY=True +CORE_URL=http://core:8080 +CORE_LOCAL_URL=http://127.0.0.1:8080 +JOBSERVICE_URL=http://jobservice:8080 +TRIVY_ADAPTER_URL=http://trivy-adapter:8080 +NOTARY_URL=http://notary-server:4443 +REGISTRY_STORAGE_PROVIDER_NAME=filesystem +READ_ONLY=false +RELOAD_KEY= +CHART_REPOSITORY_URL=http://chartmuseum:9999 +REGISTRY_CONTROLLER_URL=http://registryctl:8080 +WITH_CHARTMUSEUM=False +REGISTRY_CREDENTIAL_USERNAME=harbor_registry_user +REGISTRY_CREDENTIAL_PASSWORD=nTDpudVQRYA4rPGrDvmLvHskdd5gPcUU +CSRF_KEY=dtP7zBtDmaQR9fhuxuy5fNpbCVZfFSD4 +PERMITTED_REGISTRY_TYPES_FOR_PROXY_CACHE=docker-hub,harbor,azure-acr,aws-ecr,google-gcr,quay,docker-registry + +HTTP_PROXY= +HTTPS_PROXY= +NO_PROXY=.internal,notary-signer,registryctl,.local,nginx,chartmuseum,portal,127.0.0.1,exporter,redis,jobservice,db,core,registry,localhost,trivy-adapter,log,notary-server,postgresql + +PORT=8080 + + diff --git a/harbor/common/config/db/env b/harbor/common/config/db/env new file mode 100644 index 0000000..82fea3f --- /dev/null +++ b/harbor/common/config/db/env @@ -0,0 +1 @@ +POSTGRES_PASSWORD=mH3^tE6FPhUxk#W5iBP9FTq^AsyB8g \ No newline at end of file diff --git a/harbor/common/config/jobservice/config.yml b/harbor/common/config/jobservice/config.yml new file mode 100644 index 0000000..c268a55 --- /dev/null +++ b/harbor/common/config/jobservice/config.yml @@ -0,0 +1,36 @@ +--- +#Protocol used to serve +protocol: "http" + +#Server listening port +port: 8080 + +#Worker pool +worker_pool: + #Worker concurrency + workers: 10 + backend: "redis" + #Additional config if use 'redis' backend + redis_pool: + #redis://[arbitrary_username:password@]ipaddress:port/database_index + redis_url: redis://redis:6379/2?idle_timeout_seconds=30 + namespace: "harbor_job_service_namespace" + idle_timeout_second: 3600 +#Loggers for the running job +job_loggers: + - name: "STD_OUTPUT" # logger backend name, only support "FILE" and "STD_OUTPUT" + level: "INFO" # INFO/DEBUG/WARNING/ERROR/FATAL + - name: "FILE" + level: "INFO" + settings: # Customized settings of logger + base_dir: "/var/log/jobs" + sweeper: + duration: 1 #days + settings: # Customized settings of sweeper + work_dir: "/var/log/jobs" + +#Loggers for the job service +loggers: + - name: "STD_OUTPUT" # Same with above + level: "INFO" + diff --git a/harbor/common/config/jobservice/env b/harbor/common/config/jobservice/env new file mode 100644 index 0000000..f1564f6 --- /dev/null +++ b/harbor/common/config/jobservice/env @@ -0,0 +1,15 @@ +CORE_SECRET=Q1kOSJ2hbw3qs2Uh +REGISTRY_URL=http://registry:5000 +JOBSERVICE_SECRET=LURd3ymSGca6nuB5 +CORE_URL=http://core:8080 +REGISTRY_CONTROLLER_URL=http://registryctl:8080 +JOBSERVICE_WEBHOOK_JOB_MAX_RETRY=10 + + +HTTP_PROXY= +HTTPS_PROXY= +NO_PROXY=.internal,notary-signer,registryctl,.local,nginx,chartmuseum,portal,127.0.0.1,exporter,redis,jobservice,db,core,registry,localhost,trivy-adapter,log,notary-server,postgresql +REGISTRY_CREDENTIAL_USERNAME=harbor_registry_user +REGISTRY_CREDENTIAL_PASSWORD=nTDpudVQRYA4rPGrDvmLvHskdd5gPcUU + + diff --git a/harbor/common/config/log/logrotate.conf b/harbor/common/config/log/logrotate.conf new file mode 100644 index 0000000..97f5f93 --- /dev/null +++ b/harbor/common/config/log/logrotate.conf @@ -0,0 +1,8 @@ +/var/log/docker/*.log { + rotate 50 + size 200M + copytruncate + compress + missingok + nodateext +} \ No newline at end of file diff --git a/harbor/common/config/log/rsyslog_docker.conf b/harbor/common/config/log/rsyslog_docker.conf new file mode 100644 index 0000000..0be27a6 --- /dev/null +++ b/harbor/common/config/log/rsyslog_docker.conf @@ -0,0 +1,7 @@ +# Rsyslog configuration file for docker. + +template(name="DynaFile" type="string" string="/var/log/docker/%programname%.log") + +if $programname != "rsyslogd" then { + action(type="omfile" dynaFile="DynaFile") +} \ No newline at end of file diff --git a/harbor/common/config/nginx/nginx.conf b/harbor/common/config/nginx/nginx.conf new file mode 100644 index 0000000..a9ab535 --- /dev/null +++ b/harbor/common/config/nginx/nginx.conf @@ -0,0 +1,130 @@ +worker_processes auto; +pid /tmp/nginx.pid; + +events { + worker_connections 3096; + use epoll; + multi_accept on; +} + +http { + client_body_temp_path /tmp/client_body_temp; + proxy_temp_path /tmp/proxy_temp; + fastcgi_temp_path /tmp/fastcgi_temp; + uwsgi_temp_path /tmp/uwsgi_temp; + scgi_temp_path /tmp/scgi_temp; + tcp_nodelay on; + + # this is necessary for us to be able to disable request buffering in all cases + proxy_http_version 1.1; + + upstream core { + server core:8080; + } + + upstream portal { + server portal:8080; + } + + log_format timed_combined '$remote_addr - ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent" ' + '$request_time $upstream_response_time $pipe'; + + access_log /dev/stdout timed_combined; + + map $http_x_forwarded_proto $x_forwarded_proto { + default $http_x_forwarded_proto; + "" $scheme; + } + + server { + listen 8080; + server_tokens off; + # disable any limits to avoid HTTP 413 for large image uploads + client_max_body_size 0; + + # Add extra headers + add_header X-Frame-Options DENY; + add_header Content-Security-Policy "frame-ancestors 'none'"; + + # customized location config file can place to /etc/nginx/etc with prefix harbor.http. and suffix .conf + include /etc/nginx/conf.d/harbor.http.*.conf; + + location / { + proxy_pass http://portal/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /c/ { + proxy_pass http://core/c/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /api/ { + proxy_pass http://core/api/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /chartrepo/ { + proxy_pass http://core/chartrepo/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /v1/ { + return 404; + } + + location /v2/ { + proxy_pass http://core/v2/; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + proxy_buffering off; + proxy_request_buffering off; + + proxy_send_timeout 900; + proxy_read_timeout 900; + } + + location /service/ { + proxy_pass http://core/service/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /service/notifications { + return 404; + } + } +} diff --git a/harbor/common/config/nginx/nginx.conf~ b/harbor/common/config/nginx/nginx.conf~ new file mode 100644 index 0000000..e6f2ab8 --- /dev/null +++ b/harbor/common/config/nginx/nginx.conf~ @@ -0,0 +1,130 @@ +worker_processes auto; +pid /tmp/nginx.pid; + +events { + worker_connections 3096; + use epoll; + multi_accept on; +} + +http { + client_body_temp_path /tmp/client_body_temp; + proxy_temp_path /tmp/proxy_temp; + fastcgi_temp_path /tmp/fastcgi_temp; + uwsgi_temp_path /tmp/uwsgi_temp; + scgi_temp_path /tmp/scgi_temp; + tcp_nodelay on; + + # this is necessary for us to be able to disable request buffering in all cases + proxy_http_version 1.1; + + upstream core { + server core:8080; + } + + upstream portal { + server portal:8080; + } + + log_format timed_combined '$remote_addr - ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent" ' + '$request_time $upstream_response_time $pipe'; + + access_log /dev/stdout timed_combined; + + map $http_x_forwarded_proto $x_forwarded_proto { + default $http_x_forwarded_proto; + "" $scheme; + } + + server { + listen 8080; + server_tokens off; + # disable any limits to avoid HTTP 413 for large image uploads + client_max_body_size 0; + + # Add extra headers + add_header X-Frame-Options DENY; + add_header Content-Security-Policy "frame-ancestors 'none'"; + + # customized location config file can place to /etc/nginx/etc with prefix harbor.http. and suffix .conf + include /etc/nginx/conf.d/harbor.http.*.conf; + + location / { + proxy_pass http://portal/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /c/ { + proxy_pass http://core/c/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /api/ { + proxy_pass http://core/api/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /chartrepo/ { + proxy_pass http://core/chartrepo/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /v1/ { + return 404; + } + + location /v2/ { + proxy_pass http://core/v2/; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + proxy_buffering off; + proxy_request_buffering off; + + proxy_send_timeout 900; + proxy_read_timeout 900; + } + + location /service/ { + proxy_pass http://core/service/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /service/notifications { + return 404; + } + } +} \ No newline at end of file diff --git a/harbor/common/config/portal/nginx.conf b/harbor/common/config/portal/nginx.conf new file mode 100644 index 0000000..475fa6e --- /dev/null +++ b/harbor/common/config/portal/nginx.conf @@ -0,0 +1,38 @@ + +worker_processes auto; +pid /tmp/nginx.pid; + +events { + worker_connections 1024; +} + +http { + + client_body_temp_path /tmp/client_body_temp; + proxy_temp_path /tmp/proxy_temp; + fastcgi_temp_path /tmp/fastcgi_temp; + uwsgi_temp_path /tmp/uwsgi_temp; + scgi_temp_path /tmp/scgi_temp; + + server { + listen 8080; + server_name localhost; + + root /usr/share/nginx/html; + index index.html index.htm; + include /etc/nginx/mime.types; + + gzip on; + gzip_min_length 1000; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript; + + location / { + try_files $uri $uri/ /index.html; + } + + location = /index.html { + add_header Cache-Control "no-store, no-cache, must-revalidate"; + } + } +} \ No newline at end of file diff --git a/harbor/common/config/registry/config.yml b/harbor/common/config/registry/config.yml new file mode 100644 index 0000000..f7684b3 --- /dev/null +++ b/harbor/common/config/registry/config.yml @@ -0,0 +1,43 @@ +version: 0.1 +log: + level: info + fields: + service: registry +storage: + cache: + layerinfo: redis + filesystem: + rootdirectory: /storage + maintenance: + uploadpurging: + enabled: true + age: 168h + interval: 24h + dryrun: false + delete: + enabled: true +redis: + addr: redis:6379 + readtimeout: 10s + writetimeout: 10s + dialtimeout: 10s + password: + db: 1 + pool: + maxidle: 100 + maxactive: 500 + idletimeout: 60s +http: + addr: :5000 + secret: placeholder + debug: + addr: localhost:5001 +auth: + htpasswd: + realm: harbor-registry-basic-realm + path: /etc/registry/passwd +validation: + disabled: true +compatibility: + schema1: + enabled: true \ No newline at end of file diff --git a/harbor/common/config/registry/passwd b/harbor/common/config/registry/passwd new file mode 100644 index 0000000..b482bfa --- /dev/null +++ b/harbor/common/config/registry/passwd @@ -0,0 +1 @@ +harbor_registry_user:$2y$05$ZZgSSdASjscjf4QfvkaEHus/y62rx0h2qPoLeb/MsoG5FvUmrPfI. diff --git a/harbor/common/config/registry/root.crt b/harbor/common/config/registry/root.crt new file mode 100755 index 0000000..e69de29 diff --git a/harbor/common/config/registryctl/config.yml b/harbor/common/config/registryctl/config.yml new file mode 100644 index 0000000..2c70735 --- /dev/null +++ b/harbor/common/config/registryctl/config.yml @@ -0,0 +1,5 @@ +--- +protocol: "http" +port: 8080 +log_level: info +registry_config: "/etc/registry/config.yml" \ No newline at end of file diff --git a/harbor/common/config/registryctl/env b/harbor/common/config/registryctl/env new file mode 100644 index 0000000..046a573 --- /dev/null +++ b/harbor/common/config/registryctl/env @@ -0,0 +1,2 @@ +CORE_SECRET=Q1kOSJ2hbw3qs2Uh +JOBSERVICE_SECRET=LURd3ymSGca6nuB5 diff --git a/harbor/common/config/trivy-adapter/env b/harbor/common/config/trivy-adapter/env new file mode 100644 index 0000000..cf97f05 --- /dev/null +++ b/harbor/common/config/trivy-adapter/env @@ -0,0 +1,19 @@ +SCANNER_LOG_LEVEL=info +SCANNER_REDIS_URL=redis://redis:6379/5?idle_timeout_seconds=30 +SCANNER_STORE_REDIS_URL=redis://redis:6379/5?idle_timeout_seconds=30 +SCANNER_STORE_REDIS_NAMESPACE=harbor.scanner.trivy:store +SCANNER_JOB_QUEUE_REDIS_URL=redis://redis:6379/5?idle_timeout_seconds=30 +SCANNER_JOB_QUEUE_REDIS_NAMESPACE=harbor.scanner.trivy:job-queue +SCANNER_TRIVY_CACHE_DIR=/home/scanner/.cache/trivy +SCANNER_TRIVY_REPORTS_DIR=/home/scanner/.cache/reports +SCANNER_TRIVY_VULN_TYPE=os,library +SCANNER_TRIVY_SEVERITY=UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL +SCANNER_TRIVY_IGNORE_UNFIXED=False +SCANNER_TRIVY_SKIP_UPDATE=False +SCANNER_TRIVY_OFFLINE_SCAN=False +SCANNER_TRIVY_GITHUB_TOKEN= +SCANNER_TRIVY_INSECURE=False +SCANNER_TRIVY_TIMEOUT=5m0s +HTTP_PROXY= +HTTPS_PROXY= +NO_PROXY=.internal,notary-signer,registryctl,.local,nginx,chartmuseum,portal,127.0.0.1,exporter,redis,jobservice,db,core,registry,localhost,trivy-adapter,log,notary-server,postgresql diff --git a/harbor/docker-compose.yml b/harbor/docker-compose.yml new file mode 100644 index 0000000..8b97dd0 --- /dev/null +++ b/harbor/docker-compose.yml @@ -0,0 +1,299 @@ +version: '2.3' +services: + log: + image: goharbor/harbor-log:v2.5.1 + container_name: harbor-log + restart: always + cap_drop: + - ALL + cap_add: + - CHOWN + - DAC_OVERRIDE + - SETGID + - SETUID + volumes: + - /var/log/harbor/:/var/log/docker/:z + - type: bind + source: ./common/config/log/logrotate.conf + target: /etc/logrotate.d/logrotate.conf + - type: bind + source: ./common/config/log/rsyslog_docker.conf + target: /etc/rsyslog.d/rsyslog_docker.conf + ports: + - 127.0.0.1:1514:10514 + networks: + - harbor + registry: + image: goharbor/registry-photon:v2.5.1 + container_name: registry + restart: always + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + volumes: + - /data/registry:/storage:z + - ./common/config/registry/:/etc/registry/:z + - type: bind + source: /data/secret/registry/root.crt + target: /etc/registry/root.crt + - type: bind + source: ./common/config/shared/trust-certificates + target: /harbor_cust_cert + networks: + - harbor + depends_on: + - log + logging: + driver: "syslog" + options: + syslog-address: "tcp://localhost:1514" + tag: "registry" + registryctl: + image: goharbor/harbor-registryctl:v2.5.1 + container_name: registryctl + env_file: + - ./common/config/registryctl/env + restart: always + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + volumes: + - /data/registry:/storage:z + - ./common/config/registry/:/etc/registry/:z + - type: bind + source: ./common/config/registryctl/config.yml + target: /etc/registryctl/config.yml + - type: bind + source: ./common/config/shared/trust-certificates + target: /harbor_cust_cert + networks: + - harbor + depends_on: + - log + logging: + driver: "syslog" + options: + syslog-address: "tcp://localhost:1514" + tag: "registryctl" + postgresql: + image: goharbor/harbor-db:v2.5.1 + container_name: harbor-db + restart: always + cap_drop: + - ALL + cap_add: + - CHOWN + - DAC_OVERRIDE + - SETGID + - SETUID + volumes: + - /data/database:/var/lib/postgresql/data:z + networks: + - harbor + env_file: + - ./common/config/db/env + depends_on: + - log + logging: + driver: "syslog" + options: + syslog-address: "tcp://localhost:1514" + tag: "postgresql" + shm_size: '1gb' + core: + image: goharbor/harbor-core:v2.5.1 + container_name: core + env_file: + - ./common/config/core/env + restart: always + cap_drop: + - ALL + cap_add: + - SETGID + - SETUID + volumes: + - /data/ca_download/:/etc/core/ca/:z + - /data/:/data/:z + - ./common/config/core/certificates/:/etc/core/certificates/:z + - type: bind + source: ./common/config/core/app.conf + target: /etc/core/app.conf + - type: bind + source: /data/secret/core/private_key.pem + target: /etc/core/private_key.pem + - type: bind + source: /data/secret/keys/secretkey + target: /etc/core/key + - type: bind + source: ./common/config/shared/trust-certificates + target: /harbor_cust_cert + networks: + - harbor + depends_on: + - log + - registry + - redis + - postgresql + logging: + driver: "syslog" + options: + syslog-address: "tcp://localhost:1514" + tag: "core" + portal: + image: goharbor/harbor-portal:v2.5.1 + container_name: portal + restart: always + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + - NET_BIND_SERVICE + volumes: + - type: bind + source: ./common/config/portal/nginx.conf + target: /etc/nginx/nginx.conf + networks: + - harbor + depends_on: + - log + logging: + driver: "syslog" + options: + syslog-address: "tcp://localhost:1514" + tag: "portal" + + jobservice: + image: goharbor/harbor-jobservice:v2.5.1 + container_name: jobservice + env_file: + - ./common/config/jobservice/env + restart: always + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + volumes: + - /data/job_logs:/var/log/jobs:z + - type: bind + source: ./common/config/jobservice/config.yml + target: /etc/jobservice/config.yml + - type: bind + source: ./common/config/shared/trust-certificates + target: /harbor_cust_cert + networks: + - harbor + depends_on: + - core + logging: + driver: "syslog" + options: + syslog-address: "tcp://localhost:1514" + tag: "jobservice" + redis: + image: goharbor/redis-photon:v2.5.1 + container_name: redis + restart: always + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + volumes: + - /data/redis:/var/lib/redis + networks: + - harbor + depends_on: + - log + logging: + driver: "syslog" + options: + syslog-address: "tcp://localhost:1514" + tag: "redis" + proxy: + image: goharbor/nginx-photon:v2.5.1 + container_name: nginx + restart: always + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + - NET_BIND_SERVICE + volumes: + - ./common/config/nginx:/etc/nginx:z + - type: bind + source: ./common/config/shared/trust-certificates + target: /harbor_cust_cert + networks: + - proxy + - harbor + depends_on: + - registry + - core + - portal + - log + logging: + driver: "syslog" + options: + syslog-address: "tcp://localhost:1514" + tag: "proxy" + labels: + - "traefik.enable=true" + - "traefik.http.routers.harbor.entrypoints=web" + - "traefik.http.routers.harbor.rule=Host(`cs-registry.ddnss.de`)" + - "traefik.http.routers.harbor.middlewares=harbor" + - "traefik.http.middlewares.harbor.redirectscheme.scheme=https" + - "traefik.http.routers.harbor-secure.entrypoints=websecure" + - "traefik.http.routers.harbor-secure.rule=Host(`cs-registry.ddnss.de`)" + - "traefik.http.routers.harbor-secure.service=harbor-secure" + - "traefik.http.routers.harbor-secure.tls=true" + - "traefik.http.routers.harbor-secure.tls.certresolver=myresolver" + - "traefik.http.routers.harbor-secure.tls.domains[0].main=cs-registry.ddnss.de" + - "traefik.http.services.harbor-secure.loadbalancer.server.port=8080" + + trivy-adapter: + container_name: trivy-adapter + image: goharbor/trivy-adapter-photon:v2.5.1 + restart: always + cap_drop: + - ALL + depends_on: + - log + - redis + networks: + - harbor + volumes: + - type: bind + source: /data/trivy-adapter/trivy + target: /home/scanner/.cache/trivy + - type: bind + source: /data/trivy-adapter/reports + target: /home/scanner/.cache/reports + - type: bind + source: ./common/config/shared/trust-certificates + target: /harbor_cust_cert + logging: + driver: "syslog" + options: + syslog-address: "tcp://localhost:1514" + tag: "trivy-adapter" + env_file: + ./common/config/trivy-adapter/env +networks: + harbor: + external: false + name: harbor + proxy: + external: true diff --git a/harbor/harbor.yml b/harbor/harbor.yml new file mode 100644 index 0000000..ff33639 --- /dev/null +++ b/harbor/harbor.yml @@ -0,0 +1,247 @@ +# Configuration file of Harbor + +# The IP address or hostname to access admin UI and registry service. +# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. +hostname: harbor + +# http related config +http: + # port for http, default is 80. If https enabled, this port will redirect to https port + port: 80 + +# https related config +# https: + # https port for harbor, default is 443 + # port: 443 + # The path of cert and key files for nginx + # certificate: /your/certificate/path + # private_key: /your/private/key/path + +# # Uncomment following will enable tls communication between all harbor components +# internal_tls: +# # set enabled to true means internal tls is enabled +# enabled: true +# # put your cert and key files on dir +# dir: /etc/harbor/tls/internal + +# Uncomment external_url if you want to enable external proxy +# And when it enabled the hostname will no longer used +external_url: https://cs-registry.ddnss.de + +# The initial password of Harbor admin +# It only works in first time to install harbor +# Remember Change the admin password from UI after launching Harbor. +harbor_admin_password: j2Q2gRX@zpGYGsUZwJ@ynvnU3gw6Y* + +# Harbor DB configuration +database: + # The password for the root user of Harbor DB. Change this before any production use. + password: mH3^tE6FPhUxk#W5iBP9FTq^AsyB8g + # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained. + max_idle_conns: 100 + # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections. + # Note: the default number of connections is 1024 for postgres of harbor. + max_open_conns: 900 + +# The default data volume +data_volume: /data + +# Harbor Storage settings by default is using /data dir on local filesystem +# Uncomment storage_service setting If you want to using external storage +# storage_service: +# # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore +# # of registry's and chart repository's containers. This is usually needed when the user hosts a internal storage with self signed certificate. +# ca_bundle: + +# # storage backend, default is filesystem, options include filesystem, azure, gcs, s3, swift and oss +# # for more info about this configuration please refer https://docs.docker.com/registry/configuration/ +# filesystem: +# maxthreads: 100 +# # set disable to true when you want to disable registry redirect +# redirect: +# disabled: false + +# Trivy configuration +# +# Trivy DB contains vulnerability information from NVD, Red Hat, and many other upstream vulnerability databases. +# It is downloaded by Trivy from the GitHub release page https://github.com/aquasecurity/trivy-db/releases and cached +# in the local file system. In addition, the database contains the update timestamp so Trivy can detect whether it +# should download a newer version from the Internet or use the cached one. Currently, the database is updated every +# 12 hours and published as a new release to GitHub. +trivy: + # ignoreUnfixed The flag to display only fixed vulnerabilities + ignore_unfixed: false + # skipUpdate The flag to enable or disable Trivy DB downloads from GitHub + # + # You might want to enable this flag in test or CI/CD environments to avoid GitHub rate limiting issues. + # If the flag is enabled you have to download the `trivy-offline.tar.gz` archive manually, extract `trivy.db` and + # `metadata.json` files and mount them in the `/home/scanner/.cache/trivy/db` path. + skip_update: false + # + # The offline_scan option prevents Trivy from sending API requests to identify dependencies. + # Scanning JAR files and pom.xml may require Internet access for better detection, but this option tries to avoid it. + # For example, the offline mode will not try to resolve transitive dependencies in pom.xml when the dependency doesn't + # exist in the local repositories. It means a number of detected vulnerabilities might be fewer in offline mode. + # It would work if all the dependencies are in local. + # This option doesn’t affect DB download. You need to specify "skip-update" as well as "offline-scan" in an air-gapped environment. + offline_scan: false + # + # insecure The flag to skip verifying registry certificate + insecure: false + # github_token The GitHub access token to download Trivy DB + # + # Anonymous downloads from GitHub are subject to the limit of 60 requests per hour. Normally such rate limit is enough + # for production operations. If, for any reason, it's not enough, you could increase the rate limit to 5000 + # requests per hour by specifying the GitHub access token. For more details on GitHub rate limiting please consult + # https://developer.github.com/v3/#rate-limiting + # + # You can create a GitHub token by following the instructions in + # https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line + # + # github_token: xxx + +jobservice: + # Maximum number of job workers in job service + max_job_workers: 10 + +notification: + # Maximum retry count for webhook job + webhook_job_max_retry: 10 + +chart: + # Change the value of absolute_url to enabled can enable absolute url in chart + absolute_url: disabled + +# Log configurations +log: + # options are debug, info, warning, error, fatal + level: info + # configs for logs in local storage + local: + # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. + rotate_count: 50 + # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. + # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G + # are all valid. + rotate_size: 200M + # The directory on your host that store log + location: /var/log/harbor + + # Uncomment following lines to enable external syslog endpoint. + # external_endpoint: + # # protocol used to transmit log to external endpoint, options is tcp or udp + # protocol: tcp + # # The host of external endpoint + # host: localhost + # # Port of external endpoint + # port: 5140 + +#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! +_version: 2.5.0 + +# Uncomment external_database if using external database. +# external_database: +# harbor: +# host: harbor_db +# port: 5432 +# db_name: harbor +# username: harbor +# password: harbor_db_password +# ssl_mode: disable +# max_idle_conns: 2 +# max_open_conns: 0 +# notary_signer: +# host: notary_signer_db_host +# port: notary_signer_db_port +# db_name: notary_signer_db_name +# username: notary_signer_db_username +# password: notary_signer_db_password +# ssl_mode: disable +# notary_server: +# host: notary_server_db_host +# port: notary_server_db_port +# db_name: notary_server_db_name +# username: notary_server_db_username +# password: notary_server_db_password +# ssl_mode: disable + +# Uncomment external_redis if using external Redis server +# external_redis: +# # support redis, redis+sentinel +# # host for redis: : +# # host for redis+sentinel: +# # :,:,: +# host: redis:6379 +# password: +# # sentinel_master_set must be set to support redis+sentinel +# #sentinel_master_set: +# # db_index 0 is for core, it's unchangeable +# registry_db_index: 1 +# jobservice_db_index: 2 +# chartmuseum_db_index: 3 +# trivy_db_index: 5 +# idle_timeout_seconds: 30 + +# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert. +# uaa: +# ca_file: /path/to/ca + +# Global proxy +# Config http proxy for components, e.g. http://my.proxy.com:3128 +# Components doesn't need to connect to each others via http proxy. +# Remove component from `components` array if want disable proxy +# for it. If you want use proxy for replication, MUST enable proxy +# for core and jobservice, and set `http_proxy` and `https_proxy`. +# Add domain to the `no_proxy` field, when you want disable proxy +# for some special registry. +proxy: + http_proxy: + https_proxy: + no_proxy: + components: + - core + - jobservice + - trivy + +# metric: +# enabled: false +# port: 9090 +# path: /metrics + +# Trace related config +# only can enable one trace provider(jaeger or otel) at the same time, +# and when using jaeger as provider, can only enable it with agent mode or collector mode. +# if using jaeger collector mode, uncomment endpoint and uncomment username, password if needed +# if using jaeger agetn mode uncomment agent_host and agent_port +# trace: +# enabled: true +# # set sample_rate to 1 if you wanna sampling 100% of trace data; set 0.5 if you wanna sampling 50% of trace data, and so forth +# sample_rate: 1 +# # # namespace used to differenciate different harbor services +# # namespace: +# # # attributes is a key value dict contains user defined attributes used to initialize trace provider +# # attributes: +# # application: harbor +# # # jaeger should be 1.26 or newer. +# # jaeger: +# # endpoint: http://hostname:14268/api/traces +# # username: +# # password: +# # agent_host: hostname +# # # export trace data by jaeger.thrift in compact mode +# # agent_port: 6831 +# # otel: +# # endpoint: hostname:4318 +# # url_path: /v1/traces +# # compression: false +# # insecure: true +# # timeout: 10s + +# enable purge _upload directories +upload_purging: + enabled: true + # remove files in _upload directories which exist for a period of time, default is one week. + age: 168h + # the interval of the purge operations + interval: 24h + dryrun: false diff --git a/harbor/harbor.yml.tmpl b/harbor/harbor.yml.tmpl new file mode 100644 index 0000000..123d9ff --- /dev/null +++ b/harbor/harbor.yml.tmpl @@ -0,0 +1,247 @@ +# Configuration file of Harbor + +# The IP address or hostname to access admin UI and registry service. +# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. +hostname: reg.mydomain.com + +# http related config +http: + # port for http, default is 80. If https enabled, this port will redirect to https port + port: 80 + +# https related config +https: + # https port for harbor, default is 443 + port: 443 + # The path of cert and key files for nginx + certificate: /your/certificate/path + private_key: /your/private/key/path + +# # Uncomment following will enable tls communication between all harbor components +# internal_tls: +# # set enabled to true means internal tls is enabled +# enabled: true +# # put your cert and key files on dir +# dir: /etc/harbor/tls/internal + +# Uncomment external_url if you want to enable external proxy +# And when it enabled the hostname will no longer used +# external_url: https://reg.mydomain.com:8433 + +# The initial password of Harbor admin +# It only works in first time to install harbor +# Remember Change the admin password from UI after launching Harbor. +harbor_admin_password: Harbor12345 + +# Harbor DB configuration +database: + # The password for the root user of Harbor DB. Change this before any production use. + password: root123 + # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained. + max_idle_conns: 100 + # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections. + # Note: the default number of connections is 1024 for postgres of harbor. + max_open_conns: 900 + +# The default data volume +data_volume: /data + +# Harbor Storage settings by default is using /data dir on local filesystem +# Uncomment storage_service setting If you want to using external storage +# storage_service: +# # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore +# # of registry's and chart repository's containers. This is usually needed when the user hosts a internal storage with self signed certificate. +# ca_bundle: + +# # storage backend, default is filesystem, options include filesystem, azure, gcs, s3, swift and oss +# # for more info about this configuration please refer https://docs.docker.com/registry/configuration/ +# filesystem: +# maxthreads: 100 +# # set disable to true when you want to disable registry redirect +# redirect: +# disabled: false + +# Trivy configuration +# +# Trivy DB contains vulnerability information from NVD, Red Hat, and many other upstream vulnerability databases. +# It is downloaded by Trivy from the GitHub release page https://github.com/aquasecurity/trivy-db/releases and cached +# in the local file system. In addition, the database contains the update timestamp so Trivy can detect whether it +# should download a newer version from the Internet or use the cached one. Currently, the database is updated every +# 12 hours and published as a new release to GitHub. +trivy: + # ignoreUnfixed The flag to display only fixed vulnerabilities + ignore_unfixed: false + # skipUpdate The flag to enable or disable Trivy DB downloads from GitHub + # + # You might want to enable this flag in test or CI/CD environments to avoid GitHub rate limiting issues. + # If the flag is enabled you have to download the `trivy-offline.tar.gz` archive manually, extract `trivy.db` and + # `metadata.json` files and mount them in the `/home/scanner/.cache/trivy/db` path. + skip_update: false + # + # The offline_scan option prevents Trivy from sending API requests to identify dependencies. + # Scanning JAR files and pom.xml may require Internet access for better detection, but this option tries to avoid it. + # For example, the offline mode will not try to resolve transitive dependencies in pom.xml when the dependency doesn't + # exist in the local repositories. It means a number of detected vulnerabilities might be fewer in offline mode. + # It would work if all the dependencies are in local. + # This option doesn’t affect DB download. You need to specify "skip-update" as well as "offline-scan" in an air-gapped environment. + offline_scan: false + # + # insecure The flag to skip verifying registry certificate + insecure: false + # github_token The GitHub access token to download Trivy DB + # + # Anonymous downloads from GitHub are subject to the limit of 60 requests per hour. Normally such rate limit is enough + # for production operations. If, for any reason, it's not enough, you could increase the rate limit to 5000 + # requests per hour by specifying the GitHub access token. For more details on GitHub rate limiting please consult + # https://developer.github.com/v3/#rate-limiting + # + # You can create a GitHub token by following the instructions in + # https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line + # + # github_token: xxx + +jobservice: + # Maximum number of job workers in job service + max_job_workers: 10 + +notification: + # Maximum retry count for webhook job + webhook_job_max_retry: 10 + +chart: + # Change the value of absolute_url to enabled can enable absolute url in chart + absolute_url: disabled + +# Log configurations +log: + # options are debug, info, warning, error, fatal + level: info + # configs for logs in local storage + local: + # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. + rotate_count: 50 + # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. + # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G + # are all valid. + rotate_size: 200M + # The directory on your host that store log + location: /var/log/harbor + + # Uncomment following lines to enable external syslog endpoint. + # external_endpoint: + # # protocol used to transmit log to external endpoint, options is tcp or udp + # protocol: tcp + # # The host of external endpoint + # host: localhost + # # Port of external endpoint + # port: 5140 + +#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! +_version: 2.5.0 + +# Uncomment external_database if using external database. +# external_database: +# harbor: +# host: harbor_db_host +# port: harbor_db_port +# db_name: harbor_db_name +# username: harbor_db_username +# password: harbor_db_password +# ssl_mode: disable +# max_idle_conns: 2 +# max_open_conns: 0 +# notary_signer: +# host: notary_signer_db_host +# port: notary_signer_db_port +# db_name: notary_signer_db_name +# username: notary_signer_db_username +# password: notary_signer_db_password +# ssl_mode: disable +# notary_server: +# host: notary_server_db_host +# port: notary_server_db_port +# db_name: notary_server_db_name +# username: notary_server_db_username +# password: notary_server_db_password +# ssl_mode: disable + +# Uncomment external_redis if using external Redis server +# external_redis: +# # support redis, redis+sentinel +# # host for redis: : +# # host for redis+sentinel: +# # :,:,: +# host: redis:6379 +# password: +# # sentinel_master_set must be set to support redis+sentinel +# #sentinel_master_set: +# # db_index 0 is for core, it's unchangeable +# registry_db_index: 1 +# jobservice_db_index: 2 +# chartmuseum_db_index: 3 +# trivy_db_index: 5 +# idle_timeout_seconds: 30 + +# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert. +# uaa: +# ca_file: /path/to/ca + +# Global proxy +# Config http proxy for components, e.g. http://my.proxy.com:3128 +# Components doesn't need to connect to each others via http proxy. +# Remove component from `components` array if want disable proxy +# for it. If you want use proxy for replication, MUST enable proxy +# for core and jobservice, and set `http_proxy` and `https_proxy`. +# Add domain to the `no_proxy` field, when you want disable proxy +# for some special registry. +proxy: + http_proxy: + https_proxy: + no_proxy: + components: + - core + - jobservice + - trivy + +# metric: +# enabled: false +# port: 9090 +# path: /metrics + +# Trace related config +# only can enable one trace provider(jaeger or otel) at the same time, +# and when using jaeger as provider, can only enable it with agent mode or collector mode. +# if using jaeger collector mode, uncomment endpoint and uncomment username, password if needed +# if using jaeger agetn mode uncomment agent_host and agent_port +# trace: +# enabled: true +# # set sample_rate to 1 if you wanna sampling 100% of trace data; set 0.5 if you wanna sampling 50% of trace data, and so forth +# sample_rate: 1 +# # # namespace used to differenciate different harbor services +# # namespace: +# # # attributes is a key value dict contains user defined attributes used to initialize trace provider +# # attributes: +# # application: harbor +# # # jaeger should be 1.26 or newer. +# # jaeger: +# # endpoint: http://hostname:14268/api/traces +# # username: +# # password: +# # agent_host: hostname +# # # export trace data by jaeger.thrift in compact mode +# # agent_port: 6831 +# # otel: +# # endpoint: hostname:4318 +# # url_path: /v1/traces +# # compression: false +# # insecure: true +# # timeout: 10s + +# enable purge _upload directories +upload_purging: + enabled: true + # remove files in _upload directories which exist for a period of time, default is one week. + age: 168h + # the interval of the purge operations + interval: 24h + dryrun: false diff --git a/harbor/install.sh b/harbor/install.sh new file mode 100755 index 0000000..1cc2070 --- /dev/null +++ b/harbor/install.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +set -e + +DIR="$(cd "$(dirname "$0")" && pwd)" +source $DIR/common.sh + +set +o noglob + +usage=$'Please set hostname and other necessary attributes in harbor.yml first. DO NOT use localhost or 127.0.0.1 for hostname, because Harbor needs to be accessed by external clients. +Please set --with-notary if needs enable Notary in Harbor, and set ui_url_protocol/ssl_cert/ssl_cert_key in harbor.yml bacause notary must run under https. +Please set --with-trivy if needs enable Trivy in Harbor +Please set --with-chartmuseum if needs enable Chartmuseum in Harbor' +item=0 + +# notary is not enabled by default +with_notary=$false +# clair is deprecated +with_clair=$false +# trivy is not enabled by default +with_trivy=$false +# chartmuseum is not enabled by default +with_chartmuseum=$false + +while [ $# -gt 0 ]; do + case $1 in + --help) + note "$usage" + exit 0;; + --with-notary) + with_notary=true;; + --with-clair) + with_clair=true;; + --with-trivy) + with_trivy=true;; + --with-chartmuseum) + with_chartmuseum=true;; + *) + note "$usage" + exit 1;; + esac + shift || true +done + +if [ $with_clair ] +then + error "Clair is deprecated please remove it from installation arguments !!!" + exit 1 +fi + +workdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $workdir + +h2 "[Step $item]: checking if docker is installed ..."; let item+=1 +check_docker + +h2 "[Step $item]: checking docker-compose is installed ..."; let item+=1 +check_dockercompose + +if [ -f harbor*.tar.gz ] +then + h2 "[Step $item]: loading Harbor images ..."; let item+=1 + docker load -i ./harbor*.tar.gz +fi +echo "" + +h2 "[Step $item]: preparing environment ..."; let item+=1 +if [ -n "$host" ] +then + sed "s/^hostname: .*/hostname: $host/g" -i ./harbor.yml +fi + +h2 "[Step $item]: preparing harbor configs ..."; let item+=1 +prepare_para= +if [ $with_notary ] +then + prepare_para="${prepare_para} --with-notary" +fi +if [ $with_trivy ] +then + prepare_para="${prepare_para} --with-trivy" +fi +if [ $with_chartmuseum ] +then + prepare_para="${prepare_para} --with-chartmuseum" +fi + +./prepare $prepare_para +echo "" + +if [ -n "$(docker-compose ps -q)" ] +then + note "stopping existing Harbor instance ..." + docker-compose down -v +fi +echo "" + +h2 "[Step $item]: starting Harbor ..." +docker-compose up -d + +success $"----Harbor has been installed and started successfully.----" diff --git a/harbor/prepare b/harbor/prepare new file mode 100755 index 0000000..ccc1bc3 --- /dev/null +++ b/harbor/prepare @@ -0,0 +1,64 @@ +#!/bin/bash +set -e + +# If compiling source code this dir is harbor's make dir. +# If installing harbor via package, this dir is harbor's root dir. +if [[ -n "$HARBOR_BUNDLE_DIR" ]]; then + harbor_prepare_path=$HARBOR_BUNDLE_DIR +else + harbor_prepare_path="$( cd "$(dirname "$0")" ; pwd -P )" +fi +echo "prepare base dir is set to ${harbor_prepare_path}" + +# Clean up input dir +rm -rf ${harbor_prepare_path}/input +# Create a input dirs +mkdir -p ${harbor_prepare_path}/input +input_dir=${harbor_prepare_path}/input + +# Copy harbor.yml to input dir +if [[ ! "$1" =~ ^\-\- ]] && [ -f "$1" ] +then + cp $1 $input_dir/harbor.yml + shift +else + if [ -f "${harbor_prepare_path}/harbor.yml" ];then + cp ${harbor_prepare_path}/harbor.yml $input_dir/harbor.yml + else + echo "no config file: ${harbor_prepare_path}/harbor.yml" + exit 1 + fi +fi + +data_path=$(grep '^[^#]*data_volume:' $input_dir/harbor.yml | awk '{print $NF}') + +# If previous secretkeys exist, move it to new location +previous_secretkey_path=/data/secretkey +previous_defaultalias_path=/data/defaultalias + +if [ -f $previous_secretkey_path ]; then + mkdir -p $data_path/secret/keys + mv $previous_secretkey_path $data_path/secret/keys +fi +if [ -f $previous_defaultalias_path ]; then + mkdir -p $data_path/secret/keys + mv $previous_defaultalias_path $data_path/secret/keys +fi + + +# Create secret dir +secret_dir=${data_path}/secret +config_dir=$harbor_prepare_path/common/config + +# Run prepare script +docker run --rm -v $input_dir:/input \ + -v $data_path:/data \ + -v $harbor_prepare_path:/compose_location \ + -v $config_dir:/config \ + -v /:/hostfs \ + --privileged \ + goharbor/prepare:v2.5.1 prepare $@ + +echo "Clean up the input dir" +# Clean up input dir +rm -rf ${harbor_prepare_path}/input