From 3fa14d89a2c7fdbf021868318f75f53525c8948f Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 19 Jan 2020 23:21:54 +0100 Subject: [PATCH] [Backport] CI optimisation & add github token env var (#9875) * ci: use docker image for golangci-lint (#9737) * ci: re-ordering Drone CI for optimizing time (#9719) * ci: try re-ordering for optimizing time * ci: try re-ordering for optimizing time * ci: try re-ordering for optimizing time * ci: try re-ordering for optimizing time * ci: try re-ordering for optimizing time * ci: try re-ordering for optimizing time * ci: try offloading mysql8 to arm64 * Revert "ci: try offloading mysql8 to arm64" This reverts commit c60de5db1cf8b5984c3014a57da6490f06c8d980. * ci: try offloading pgsql to arm64 * ci: activate ldap on arm64 * ci: test mysql8 in place pgsql arm64 * chore: clean un-needed move * typo * ci: revert runnning mysql on arm64 * ci: run compliance on arm * chore: limit change * chore: readd maybe need for release fetch-tags * ci: remove docker-linux-amd64-dry-run * ci: remove docker-linux-amd64-dry-run * Revert "ci: remove docker-linux-amd64-dry-run" This reverts commit 0715f65b11c37869359aaaa5d22901da512e8184. Co-authored-by: zeripath Co-authored-by: Lunny Xiao * ci: use new mssql image (#9720) Co-authored-by: Lunny Xiao * ci: run notify on arm64 (#9762) * ci: run notify on arm64 Free one jobs on amrd64 * Update .drone.yml * Update .drone.yml * Update .drone.yml Based on: https://github.com/appleboy/drone-discord/blob/master/.drone.yml#L339 * improve trigger Co-authored-by: techknowlogick * ci: move some integration tests on arm64 (#9747) * tests: configure github remaining limit + read token (#9800) * ci: configure remaining github limmit * prepend with github since package is common to all migrations * add RefreshRate * Update github.go * add missing space * go fmt * Read env variable GITHUB_READ_TOKEN for token * Update .drone.yml * ci: simplify tag/release by always running coverage (#9774) * ci: simplify tag/release by always running coverage * use mod and vendor for unit test coverage * remove not needed lfs for unit test * use arm drone agent for docs (#9776) * run translations pipeline on arm server (#9865) * add git-check to Makefile Co-authored-by: Antoine GIRARD Co-authored-by: zeripath Co-authored-by: Lunny Xiao Co-authored-by: techknowlogick --- .drone.yml | 396 +++++++++++++++--------------- Makefile | 11 +- modules/migrations/github.go | 17 +- modules/migrations/github_test.go | 7 +- 4 files changed, 222 insertions(+), 209 deletions(-) diff --git a/.drone.yml b/.drone.yml index 7a844af8e..422319cfe 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,59 +1,16 @@ --- kind: pipeline -name: testing +name: compliance platform: os: linux - arch: amd64 + arch: arm64 workspace: base: /go path: src/code.gitea.io/gitea -services: - - name: mysql - pull: default - image: mysql:5.7 - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: yes - MYSQL_DATABASE: test - - - name: mysql8 - pull: default - image: mysql:8.0 - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: yes - MYSQL_DATABASE: testgitea - - - name: pgsql - pull: default - image: postgres:9.5 - environment: - POSTGRES_DB: test - - - name: mssql - pull: default - image: microsoft/mssql-server-linux:latest - environment: - ACCEPT_EULA: Y - MSSQL_PID: Standard - SA_PASSWORD: MwantsaSecurePassword1 - - - name: ldap - pull: default - image: gitea/test-openldap:latest - steps: - - name: fetch-tags - pull: default - image: docker:git - commands: - - git fetch --tags --force - when: - event: - exclude: - - pull_request - - name: pre-build pull: always image: node:10 # this step is kept at the lowest version of node that we support @@ -68,7 +25,6 @@ steps: GO111MODULE: on GOPROXY: off commands: - - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs - go build -mod=vendor -o gitea_no_gcc # test if build succeeds without the sqlite tag - name: build-linux-386 @@ -80,98 +36,94 @@ steps: GOOS: linux GOARCH: 386 commands: - - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs - go build -mod=vendor -o gitea_linux_386 # test if compatible with 32 bit - - name: build + - name: check pull: always image: golang:1.13 commands: - - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs - make clean - make golangci-lint - make revive - make swagger-check - make swagger-validate - make test-vendor - - make build environment: GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not GOSUMDB: sum.golang.org TAGS: bindata sqlite sqlite_unlock_notify - - name: unit-test - pull: always - image: golang:1.13 - commands: - - make unit-test-coverage +--- +kind: pipeline +name: testing-amd64 + +platform: + os: linux + arch: amd64 + +workspace: + base: /go + path: src/code.gitea.io/gitea + +services: + - name: mysql + pull: default + image: mysql:5.7 environment: - GOPROXY: off - TAGS: bindata sqlite sqlite_unlock_notify - depends_on: - - build - when: - branch: - - master - event: - - push - - pull_request + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_DATABASE: test - - name: release-test - pull: always - image: golang:1.13 - commands: - - make test + - name: mysql8 + pull: default + image: mysql:8.0 environment: - GOPROXY: off - TAGS: bindata sqlite sqlite_unlock_notify - depends_on: - - build - when: - branch: - - "release/*" - event: - - push - - pull_request + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_DATABASE: testgitea - - name: tag-pre-condition - pull: always - image: alpine/git + - name: mssql + pull: default + image: mcr.microsoft.com/mssql/server:latest + environment: + ACCEPT_EULA: Y + MSSQL_PID: Standard + SA_PASSWORD: MwantsaSecurePassword1 + + - name: ldap + pull: default + image: gitea/test-openldap:latest + +steps: + - name: fetch-tags + pull: default + image: docker:git commands: - - git update-ref refs/heads/tag_test ${DRONE_COMMIT_SHA} - depends_on: - - build + - git fetch --tags --force when: event: - - tag + exclude: + - pull_request - - name: tag-test + - name: build pull: always image: golang:1.13 commands: - - make test + - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs + - make build environment: - GOPROXY: off - TAGS: bindata - depends_on: - - tag-pre-condition - when: - event: - - tag + GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not + GOSUMDB: sum.golang.org + TAGS: bindata sqlite sqlite_unlock_notify - - name: test-sqlite + - name: unit-test pull: always image: golang:1.13 commands: - - "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash" - - apt-get install -y git-lfs - - timeout -s ABRT 20m make test-sqlite-migration - - timeout -s ABRT 20m make test-sqlite + - make unit-test-coverage environment: GOPROXY: off - TAGS: bindata - depends_on: - - build + TAGS: bindata sqlite sqlite_unlock_notify + GITHUB_READ_TOKEN: + from_secret: github_read_token - name: test-mysql pull: always @@ -187,30 +139,6 @@ steps: TEST_LDAP: 1 depends_on: - build - when: - branch: - - master - event: - - push - - pull_request - - - name: tag-test-mysql - pull: always - image: golang:1.13 - commands: - - "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash" - - apt-get install -y git-lfs - - timeout -s ABRT 20m make test-mysql-migration - - timeout -s ABRT 20m make test-mysql - environment: - GOPROXY: off - TAGS: bindata - TEST_LDAP: 1 - depends_on: - - build - when: - event: - - tag - name: test-mysql8 pull: always @@ -227,21 +155,6 @@ steps: depends_on: - build - - name: test-pgsql - pull: always - image: golang:1.13 - commands: - - "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash" - - apt-get install -y git-lfs - - timeout -s ABRT 20m make test-pgsql-migration - - timeout -s ABRT 20m make test-pgsql - environment: - GOPROXY: off - TAGS: bindata - TEST_LDAP: 1 - depends_on: - - build - - name: test-mssql pull: always image: golang:1.13 @@ -293,13 +206,89 @@ steps: - push - pull_request + + +--- +kind: pipeline +name: testing-arm64 + +platform: + os: linux + arch: arm64 + +workspace: + base: /go + path: src/code.gitea.io/gitea + +services: + - name: pgsql + pull: default + image: postgres:9.5 + environment: + POSTGRES_DB: test + + - name: ldap + pull: default + image: gitea/test-openldap:latest + +steps: + - name: fetch-tags + pull: default + image: docker:git + commands: + - git fetch --tags --force + when: + event: + exclude: + - pull_request + + - name: build + pull: always + image: golang:1.13 + commands: + - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs + - make build + environment: + GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not + GOSUMDB: sum.golang.org + TAGS: bindata sqlite sqlite_unlock_notify + + - name: test-sqlite + pull: always + image: golang:1.13 + commands: + - "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash" + - apt-get install -y git-lfs + - timeout -s ABRT 20m make test-sqlite-migration + - timeout -s ABRT 20m make test-sqlite + environment: + GOPROXY: off + TAGS: bindata + depends_on: + - build + + - name: test-pgsql + pull: always + image: golang:1.13 + commands: + - "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash" + - apt-get install -y git-lfs + - timeout -s ABRT 20m make test-pgsql-migration + - timeout -s ABRT 20m make test-pgsql + environment: + GOPROXY: off + TAGS: bindata + TEST_LDAP: 1 + depends_on: + - build + --- kind: pipeline name: translations platform: os: linux - arch: amd64 + arch: arm64 workspace: base: /go @@ -378,7 +367,8 @@ trigger: - push depends_on: - - testing + - testing-amd64 + - testing-arm64 - translations steps: @@ -476,7 +466,8 @@ trigger: - tag depends_on: - - testing + - testing-arm64 + - testing-amd64 steps: - name: fetch-tags @@ -545,17 +536,14 @@ name: docs platform: os: linux - arch: amd64 - -workspace: - base: /go - path: src/code.gitea.io/gitea + arch: arm64 steps: - name: build-docs pull: always - image: webhippie/hugo:latest + image: plugins/hugo:latest commands: + - apk add --no-cache make bash curl - cd docs - make trans-copy - make clean @@ -563,7 +551,7 @@ steps: - name: publish-docs pull: always - image: lucap/drone-netlify:latest + image: techknowlogick/drone-netlify:latest settings: path: docs/public/ site_id: d2260bae-7861-4c02-8646-8f6440b12672 @@ -578,7 +566,7 @@ steps: --- kind: pipeline -name: docker-linux-amd64 +name: docker-linux-amd64-release platform: os: linux @@ -589,13 +577,13 @@ workspace: path: src/code.gitea.io/gitea depends_on: - - testing + - testing-amd64 + - testing-arm64 trigger: ref: - refs/heads/master - "refs/tags/**" - - "refs/pull/**" steps: - name: fetch-tags @@ -603,23 +591,6 @@ steps: image: docker:git commands: - git fetch --tags --force - when: - event: - exclude: - - pull_request - - - name: dryrun - pull: always - image: plugins/docker:linux-amd64 - settings: - dry_run: true - repo: gitea/gitea - tags: linux-amd64 - build_args: - - GOPROXY=off - when: - event: - - pull_request - name: publish pull: always @@ -641,7 +612,7 @@ steps: --- kind: pipeline -name: docker-linux-arm64 +name: docker-linux-arm64-dry-run platform: os: linux @@ -652,25 +623,13 @@ workspace: path: src/code.gitea.io/gitea depends_on: - - testing + - compliance trigger: ref: - - refs/heads/master - - "refs/tags/**" - "refs/pull/**" steps: - - name: fetch-tags - pull: default - image: docker:git - commands: - - git fetch --tags --force - when: - event: - exclude: - - pull_request - - name: dryrun pull: always image: plugins/docker:linux-arm64 @@ -684,6 +643,33 @@ steps: event: - pull_request +--- +kind: pipeline +name: docker-linux-arm64-release + +platform: + os: linux + arch: arm64 + +workspace: + base: /go + path: src/code.gitea.io/gitea + +depends_on: + - testing-amd64 + - testing-arm64 + +trigger: + ref: + - refs/heads/master + - "refs/tags/**" +steps: + - name: fetch-tags + pull: default + image: docker:git + commands: + - git fetch --tags --force + - name: publish pull: always image: plugins/docker:linux-arm64 @@ -729,45 +715,49 @@ trigger: - "refs/tags/**" depends_on: - - docker-linux-amd64 - - docker-linux-arm64 + - docker-linux-amd64-release + - docker-linux-arm64-release --- kind: pipeline -name: notify +name: notifications platform: os: linux - arch: amd64 - -workspace: - base: /go - path: src/code.gitea.io/gitea + arch: arm64 clone: disable: true -when: +trigger: + branch: + - master + - "release/*" + event: + - push + - tag status: - success - failure depends_on: - - testing + - testing-amd64 + - testing-arm64 - translations - release-version - release-master - - docker-linux-amd64 - - docker-linux-arm64 + - docker-linux-amd64-release + - docker-linux-arm64-release - docker-manifest - docs steps: - name: discord pull: always - image: appleboy/drone-discord:1.0.0 - environment: - DISCORD_WEBHOOK_ID: + image: appleboy/drone-discord:1.2.4 + settings: + message: "{{#success build.status}} āœ… Build #{{build.number}} of `{{repo.name}}` succeeded.\n\nšŸ“ Commit by {{commit.author}} on `{{commit.branch}}`:\n``` {{commit.message}} ```\n\nšŸŒ {{ build.link }} {{else}} āŒ Build #{{build.number}} of `{{repo.name}}` failed.\n\nšŸ“ Commit by {{commit.author}} on `{{commit.branch}}`:\n``` {{commit.message}} ```\n\nšŸŒ {{ build.link }} {{/success}}\n" + webhook_id: from_secret: discord_webhook_id - DISCORD_WEBHOOK_TOKEN: + webhook_token: from_secret: discord_webhook_token diff --git a/Makefile b/Makefile index 8da85c7a8..7f5ba4534 100644 --- a/Makefile +++ b/Makefile @@ -119,6 +119,13 @@ go-check: exit 1; \ fi +.PHONY: git-check +git-check: + @if git lfs >/dev/null 2>&1 ; then : ; else \ + echo "Gitea requires git with lfs support to run tests." ; \ + exit 1; \ + fi + .PHONY: node-check node-check: $(eval NODE_VERSION := $(shell printf "%03d%03d%03d" $(shell node -v | grep -Eo '[0-9]+\.?[0-9]+?\.?[0-9]?' | tr '.' ' ');)) @@ -233,7 +240,7 @@ coverage: .PHONY: unit-test-coverage unit-test-coverage: - $(GO) test -tags='sqlite sqlite_unlock_notify' -cover -coverprofile coverage.out $(PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1 + GO111MODULE=on $(GO) test -mod=vendor -tags='sqlite sqlite_unlock_notify' -cover -coverprofile coverage.out $(PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1 .PHONY: vendor vendor: @@ -376,7 +383,7 @@ integrations.mssql.test: $(GO_SOURCES) integrations.sqlite.test: $(GO_SOURCES) GO111MODULE=on $(GO) test -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags 'sqlite sqlite_unlock_notify' -integrations.cover.test: $(GO_SOURCES) +integrations.cover.test: git-check $(GO_SOURCES) GO111MODULE=on $(GO) test -mod=vendor -c code.gitea.io/gitea/integrations -coverpkg $(shell echo $(PACKAGES) | tr ' ' ',') -o integrations.cover.test .PHONY: migrations.mysql.test diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 9183c9318..52344d017 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -24,6 +24,8 @@ import ( var ( _ base.Downloader = &GithubDownloaderV3{} _ base.DownloaderFactory = &GithubDownloaderV3Factory{} + // GithubLimitRateRemaining limit to wait for new rate to apply + GithubLimitRateRemaining = 0 ) func init() { @@ -115,7 +117,7 @@ func (g *GithubDownloaderV3) SetContext(ctx context.Context) { } func (g *GithubDownloaderV3) sleep() { - for g.rate != nil && g.rate.Remaining <= 0 { + for g.rate != nil && g.rate.Remaining <= GithubLimitRateRemaining { timer := time.NewTimer(time.Until(g.rate.Reset.Time)) select { case <-g.ctx.Done(): @@ -124,13 +126,22 @@ func (g *GithubDownloaderV3) sleep() { case <-timer.C: } - rates, _, err := g.client.RateLimits(g.ctx) + err := g.RefreshRate() if err != nil { log.Error("g.client.RateLimits: %s", err) } + } +} - g.rate = rates.GetCore() +// RefreshRate update the current rate (doesn't count in rate limit) +func (g *GithubDownloaderV3) RefreshRate() error { + rates, _, err := g.client.RateLimits(g.ctx) + if err != nil { + return err } + + g.rate = rates.GetCore() + return nil } // GetRepoInfo returns a repository information diff --git a/modules/migrations/github_test.go b/modules/migrations/github_test.go index bf71ab475..0ff939dee 100644 --- a/modules/migrations/github_test.go +++ b/modules/migrations/github_test.go @@ -6,6 +6,7 @@ package migrations import ( + "os" "testing" "time" @@ -62,7 +63,11 @@ func assertLabelEqual(t *testing.T, name, color, description string, label *base } func TestGitHubDownloadRepo(t *testing.T) { - downloader := NewGithubDownloaderV3("", "", "go-gitea", "test_repo") + GithubLimitRateRemaining = 3 //Wait at 3 remaining since we could have 3 CI in // + downloader := NewGithubDownloaderV3(os.Getenv("GITHUB_READ_TOKEN"), "", "go-gitea", "test_repo") + err := downloader.RefreshRate() + assert.NoError(t, err) + repo, err := downloader.GetRepoInfo() assert.NoError(t, err) assert.EqualValues(t, &base.Repository{