[Backport] CI optimisation & add github token env var (#9875)

* ci: use docker image for golangci-lint (#9737) * ci: re-ordering Drone CI for optimizing time (#9719) * ci: try re-ordering for optimizing time * ci: try re-ordering for optimizing time * ci: try re-ordering for optimizing time * ci: try re-ordering for optimizing time * ci: try re-ordering for optimizing time * ci: try re-ordering for optimizing time * ci: try offloading mysql8 to arm64 * Revert "ci: try offloading mysql8 to arm64" This reverts commit c60de5db1cf8b5984c3014a57da6490f06c8d980. * ci: try offloading pgsql to arm64 * ci: activate ldap on arm64 * ci: test mysql8 in place pgsql arm64 * chore: clean un-needed move * typo * ci: revert runnning mysql on arm64 * ci: run compliance on arm * chore: limit change * chore: readd maybe need for release fetch-tags * ci: remove docker-linux-amd64-dry-run * ci: remove docker-linux-amd64-dry-run * Revert "ci: remove docker-linux-amd64-dry-run" This reverts commit 0715f65b11c37869359aaaa5d22901da512e8184. Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com> * ci: use new mssql image (#9720) Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com> * ci: run notify on arm64 (#9762) * ci: run notify on arm64 Free one jobs on amrd64 * Update .drone.yml * Update .drone.yml * Update .drone.yml Based on: https://github.com/appleboy/drone-discord/blob/master/.drone.yml#L339 * improve trigger Co-authored-by: techknowlogick <matti@mdranta.net> * ci: move some integration tests on arm64 (#9747) * tests: configure github remaining limit + read token (#9800) * ci: configure remaining github limmit * prepend with github since package is common to all migrations * add RefreshRate * Update github.go * add missing space * go fmt * Read env variable GITHUB_READ_TOKEN for token * Update .drone.yml * ci: simplify tag/release by always running coverage (#9774) * ci: simplify tag/release by always running coverage * use mod and vendor for unit test coverage * remove not needed lfs for unit test * use arm drone agent for docs (#9776) * run translations pipeline on arm server (#9865) * add git-check to Makefile Co-authored-by: Antoine GIRARD <sapk@users.noreply.github.com> Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com> Co-authored-by: techknowlogick <matti@mdranta.net>
4 years ago · 3fa14d89a2
parent bcb722daec
commit 3fa14d89a2
4 changed files with 222 additions and 209 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -1,59 +1,16 @@
 ---
 kind: pipeline
-name: testing
+name: compliance

 platform:
  os: linux
-  arch: amd64
+  arch: arm64

 workspace:
  base: /go
  path: src/code.gitea.io/gitea

-services:
-  - name: mysql
-    pull: default
-    image: mysql:5.7
-    environment:
-      MYSQL_ALLOW_EMPTY_PASSWORD: yes
-      MYSQL_DATABASE: test
-
-  - name: mysql8
-    pull: default
-    image: mysql:8.0
-    environment:
-      MYSQL_ALLOW_EMPTY_PASSWORD: yes
-      MYSQL_DATABASE: testgitea
-
-  - name: pgsql
-    pull: default
-    image: postgres:9.5
-    environment:
-      POSTGRES_DB: test
-
-  - name: mssql
-    pull: default
-    image: microsoft/mssql-server-linux:latest
-    environment:
-      ACCEPT_EULA: Y
-      MSSQL_PID: Standard
-      SA_PASSWORD: MwantsaSecurePassword1
-
-  - name: ldap
-    pull: default
-    image: gitea/test-openldap:latest
-
 steps:
-  - name: fetch-tags
-    pull: default
-    image: docker:git
-    commands:
-      - git fetch --tags --force
-    when:
-      event:
-        exclude:
-          - pull_request
-
  - name: pre-build
    pull: always
    image: node:10 # this step is kept at the lowest version of node that we support
@ -68,7 +25,6 @@ steps:
      GO111MODULE: on
      GOPROXY: off
    commands:
-      - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
      - go build -mod=vendor -o gitea_no_gcc # test if build succeeds without the sqlite tag

  - name: build-linux-386
@ -80,98 +36,94 @@ steps:
      GOOS: linux
      GOARCH: 386
    commands:
-      - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
      - go build -mod=vendor -o gitea_linux_386 # test if compatible with 32 bit

-  - name: build
+  - name: check
    pull: always
    image: golang:1.13
    commands:
-      - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
      - make clean
      - make golangci-lint
      - make revive
      - make swagger-check
      - make swagger-validate
      - make test-vendor
-      - make build
    environment:
      GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
      GOSUMDB: sum.golang.org
      TAGS: bindata sqlite sqlite_unlock_notify

-  - name: unit-test
-    pull: always
-    image: golang:1.13
-    commands:
-      - make unit-test-coverage
+---
+kind: pipeline
+name: testing-amd64
+
+platform:
+  os: linux
+  arch: amd64
+
+workspace:
+  base: /go
+  path: src/code.gitea.io/gitea
+
+services:
+  - name: mysql
+    pull: default
+    image: mysql:5.7
    environment:
-      GOPROXY: off
-      TAGS: bindata sqlite sqlite_unlock_notify
-    depends_on:
-      - build
-    when:
-      branch:
-        - master
-      event:
-        - push
-        - pull_request
+      MYSQL_ALLOW_EMPTY_PASSWORD: yes
+      MYSQL_DATABASE: test

-  - name: release-test
-    pull: always
-    image: golang:1.13
-    commands:
-      - make test
+  - name: mysql8
+    pull: default
+    image: mysql:8.0
    environment:
-      GOPROXY: off
-      TAGS: bindata sqlite sqlite_unlock_notify
-    depends_on:
-      - build
-    when:
-      branch:
-        - "release/*"
-      event:
-        - push
-        - pull_request
+      MYSQL_ALLOW_EMPTY_PASSWORD: yes
+      MYSQL_DATABASE: testgitea

-  - name: tag-pre-condition
-    pull: always
-    image: alpine/git
+  - name: mssql
+    pull: default
+    image: mcr.microsoft.com/mssql/server:latest
+    environment:
+      ACCEPT_EULA: Y
+      MSSQL_PID: Standard
+      SA_PASSWORD: MwantsaSecurePassword1
+
+  - name: ldap
+    pull: default
+    image: gitea/test-openldap:latest
+
+steps:
+  - name: fetch-tags
+    pull: default
+    image: docker:git
    commands:
-      - git update-ref refs/heads/tag_test ${DRONE_COMMIT_SHA}
-    depends_on:
-      - build
+      - git fetch --tags --force
    when:
      event:
-        - tag
+        exclude:
+          - pull_request

-  - name: tag-test
+  - name: build
    pull: always
    image: golang:1.13
    commands:
-      - make test
+      - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
+      - make build
    environment:
-      GOPROXY: off
-      TAGS: bindata
-    depends_on:
-      - tag-pre-condition
-    when:
-      event:
-        - tag
+      GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
+      GOSUMDB: sum.golang.org
+      TAGS: bindata sqlite sqlite_unlock_notify

-  - name: test-sqlite
+  - name: unit-test
    pull: always
    image: golang:1.13
    commands:
-      - "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
-      - apt-get install -y git-lfs
-      - timeout -s ABRT 20m make test-sqlite-migration
-      - timeout -s ABRT 20m make test-sqlite
+      - make unit-test-coverage
    environment:
      GOPROXY: off
-      TAGS: bindata
-    depends_on:
-      - build
+      TAGS: bindata sqlite sqlite_unlock_notify
+      GITHUB_READ_TOKEN:
+        from_secret: github_read_token

  - name: test-mysql
    pull: always
@ -187,30 +139,6 @@ steps:
      TEST_LDAP: 1
    depends_on:
      - build
-    when:
-      branch:
-        - master
-      event:
-        - push
-        - pull_request
-
-  - name: tag-test-mysql
-    pull: always
-    image: golang:1.13
-    commands:
-      - "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
-      - apt-get install -y git-lfs
-      - timeout -s ABRT 20m make test-mysql-migration
-      - timeout -s ABRT 20m make test-mysql
-    environment:
-      GOPROXY: off
-      TAGS: bindata
-      TEST_LDAP: 1
-    depends_on:
-      - build
-    when:
-      event:
-        - tag

  - name: test-mysql8
    pull: always
@ -227,21 +155,6 @@ steps:
    depends_on:
      - build

-  - name: test-pgsql
-    pull: always
-    image: golang:1.13
-    commands:
-      - "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
-      - apt-get install -y git-lfs
-      - timeout -s ABRT 20m make test-pgsql-migration
-      - timeout -s ABRT 20m make test-pgsql
-    environment:
-      GOPROXY: off
-      TAGS: bindata
-      TEST_LDAP: 1
-    depends_on:
-      - build
-
  - name: test-mssql
    pull: always
    image: golang:1.13
@ -293,13 +206,89 @@ steps:
        - push
        - pull_request

+
+
+---
+kind: pipeline
+name: testing-arm64
+
+platform:
+  os: linux
+  arch: arm64
+
+workspace:
+  base: /go
+  path: src/code.gitea.io/gitea
+
+services:
+  - name: pgsql
+    pull: default
+    image: postgres:9.5
+    environment:
+      POSTGRES_DB: test
+
+  - name: ldap
+    pull: default
+    image: gitea/test-openldap:latest
+
+steps:
+  - name: fetch-tags
+    pull: default
+    image: docker:git
+    commands:
+      - git fetch --tags --force
+    when:
+      event:
+        exclude:
+          - pull_request
+
+  - name: build
+    pull: always
+    image: golang:1.13
+    commands:
+      - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
+      - make build
+    environment:
+      GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
+      GOSUMDB: sum.golang.org
+      TAGS: bindata sqlite sqlite_unlock_notify
+
+  - name: test-sqlite
+    pull: always
+    image: golang:1.13
+    commands:
+      - "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
+      - apt-get install -y git-lfs
+      - timeout -s ABRT 20m make test-sqlite-migration
+      - timeout -s ABRT 20m make test-sqlite
+    environment:
+      GOPROXY: off
+      TAGS: bindata
+    depends_on:
+      - build
+
+  - name: test-pgsql
+    pull: always
+    image: golang:1.13
+    commands:
+      - "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
+      - apt-get install -y git-lfs
+      - timeout -s ABRT 20m make test-pgsql-migration
+      - timeout -s ABRT 20m make test-pgsql
+    environment:
+      GOPROXY: off
+      TAGS: bindata
+      TEST_LDAP: 1
+    depends_on:
+      - build
+
 ---
 kind: pipeline
 name: translations

 platform:
  os: linux
-  arch: amd64
+  arch: arm64

 workspace:
  base: /go
@ -378,7 +367,8 @@ trigger:
    - push

 depends_on:
-  - testing
+  - testing-amd64
+  - testing-arm64
  - translations

 steps:
@ -476,7 +466,8 @@ trigger:
    - tag

 depends_on:
-  - testing
+  - testing-arm64
+  - testing-amd64

 steps:
  - name: fetch-tags
@ -545,17 +536,14 @@ name: docs

 platform:
  os: linux
-  arch: amd64
-
-workspace:
-  base: /go
-  path: src/code.gitea.io/gitea
+  arch: arm64

 steps:
  - name: build-docs
    pull: always
-    image: webhippie/hugo:latest
+    image: plugins/hugo:latest
    commands:
+      - apk add --no-cache make bash curl
      - cd docs
      - make trans-copy
      - make clean
@ -563,7 +551,7 @@ steps:

  - name: publish-docs
    pull: always
-    image: lucap/drone-netlify:latest
+    image: techknowlogick/drone-netlify:latest
    settings:
      path: docs/public/
      site_id: d2260bae-7861-4c02-8646-8f6440b12672
@ -578,7 +566,7 @@ steps:

 ---
 kind: pipeline
-name: docker-linux-amd64
+name: docker-linux-amd64-release

 platform:
  os: linux
@ -589,13 +577,13 @@ workspace:
  path: src/code.gitea.io/gitea

 depends_on:
-  - testing
+  - testing-amd64
+  - testing-arm64

 trigger:
  ref:
  - refs/heads/master
  - "refs/tags/**"
-  - "refs/pull/**"

 steps:
  - name: fetch-tags
@ -603,23 +591,6 @@ steps:
    image: docker:git
    commands:
      - git fetch --tags --force
-    when:
-      event:
-        exclude:
-          - pull_request
-
-  - name: dryrun
-    pull: always
-    image: plugins/docker:linux-amd64
-    settings:
-      dry_run: true
-      repo: gitea/gitea
-      tags: linux-amd64
-      build_args:
-        - GOPROXY=off
-    when:
-      event:
-        - pull_request

  - name: publish
    pull: always
@ -641,7 +612,7 @@ steps:

 ---
 kind: pipeline
-name: docker-linux-arm64
+name: docker-linux-arm64-dry-run

 platform:
  os: linux
@ -652,25 +623,13 @@ workspace:
  path: src/code.gitea.io/gitea

 depends_on:
-  - testing
+  - compliance

 trigger:
  ref:
-  - refs/heads/master
-  - "refs/tags/**"
  - "refs/pull/**"

 steps:
-  - name: fetch-tags
-    pull: default
-    image: docker:git
-    commands:
-      - git fetch --tags --force
-    when:
-      event:
-        exclude:
-          - pull_request
-
  - name: dryrun
    pull: always
    image: plugins/docker:linux-arm64
@ -684,6 +643,33 @@ steps:
      event:
        - pull_request

+---
+kind: pipeline
+name: docker-linux-arm64-release
+
+platform:
+  os: linux
+  arch: arm64
+
+workspace:
+  base: /go
+  path: src/code.gitea.io/gitea
+
+depends_on:
+  - testing-amd64
+  - testing-arm64
+
+trigger:
+  ref:
+  - refs/heads/master
+  - "refs/tags/**"
+steps:
+  - name: fetch-tags
+    pull: default
+    image: docker:git
+    commands:
+      - git fetch --tags --force
+
  - name: publish
    pull: always
    image: plugins/docker:linux-arm64
@ -729,45 +715,49 @@ trigger:
  - "refs/tags/**"

 depends_on:
-  - docker-linux-amd64
-  - docker-linux-arm64
+  - docker-linux-amd64-release
+  - docker-linux-arm64-release

 ---
 kind: pipeline
-name: notify
+name: notifications

 platform:
  os: linux
-  arch: amd64
-
-workspace:
-  base: /go
-  path: src/code.gitea.io/gitea
+  arch: arm64

 clone:
  disable: true

-when:
+trigger:
+  branch:
+    - master
+    - "release/*"
+  event:
+    - push
+    - tag
  status:
    - success
    - failure

 depends_on:
-  - testing
+  - testing-amd64
+  - testing-arm64
  - translations
  - release-version
  - release-master
-  - docker-linux-amd64
-  - docker-linux-arm64
+  - docker-linux-amd64-release
+  - docker-linux-arm64-release
  - docker-manifest
  - docs

 steps:
  - name: discord
    pull: always
-    image: appleboy/drone-discord:1.0.0
-    environment:
-      DISCORD_WEBHOOK_ID:
+    image: appleboy/drone-discord:1.2.4
+    settings:
+      message: "{{#success build.status}} ✅  Build #{{build.number}} of `{{repo.name}}` succeeded.\n\n📝 Commit by {{commit.author}} on `{{commit.branch}}`:\n``` {{commit.message}} ```\n\n🌐 {{ build.link }} {{else}} ❌  Build #{{build.number}} of `{{repo.name}}` failed.\n\n📝 Commit by {{commit.author}} on `{{commit.branch}}`:\n``` {{commit.message}} ```\n\n🌐 {{ build.link }} {{/success}}\n"
+      webhook_id:
        from_secret: discord_webhook_id
-      DISCORD_WEBHOOK_TOKEN:
+      webhook_token:
        from_secret: discord_webhook_token
--- a/11
+++ b/11
@ -119,6 +119,13 @@ go-check:
 		exit 1; \
 	fi

+.PHONY: git-check
+git-check:
+	@if git lfs >/dev/null 2>&1 ; then : ; else \
+		echo "Gitea requires git with lfs support to run tests." ; \
+		exit 1; \
+	fi
+
 .PHONY: node-check
 node-check:
 	$(eval NODE_VERSION := $(shell printf "%03d%03d%03d" $(shell node -v | grep -Eo '[0-9]+\.?[0-9]+?\.?[0-9]?' | tr '.' ' ');))
@ -233,7 +240,7 @@ coverage:

 .PHONY: unit-test-coverage
 unit-test-coverage:
-	$(GO) test -tags='sqlite sqlite_unlock_notify' -cover -coverprofile coverage.out $(PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
+	GO111MODULE=on $(GO) test -mod=vendor -tags='sqlite sqlite_unlock_notify' -cover -coverprofile coverage.out $(PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1

 .PHONY: vendor
 vendor:
@ -376,7 +383,7 @@ integrations.mssql.test: $(GO_SOURCES)
 integrations.sqlite.test: $(GO_SOURCES)
 	GO111MODULE=on $(GO) test -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags 'sqlite sqlite_unlock_notify'

-integrations.cover.test: $(GO_SOURCES)
+integrations.cover.test: git-check $(GO_SOURCES)
 	GO111MODULE=on $(GO) test -mod=vendor -c code.gitea.io/gitea/integrations -coverpkg $(shell echo $(PACKAGES) | tr ' ' ',') -o integrations.cover.test

 .PHONY: migrations.mysql.test
--- a/modules/migrations/github.go
+++ b/modules/migrations/github.go
@ -24,6 +24,8 @@ import (
 var (
 	_ base.Downloader        = &GithubDownloaderV3{}
 	_ base.DownloaderFactory = &GithubDownloaderV3Factory{}
+	// GithubLimitRateRemaining limit to wait for new rate to apply
+	GithubLimitRateRemaining = 0
 )

 func init() {
@ -115,7 +117,7 @@ func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
 }

 func (g *GithubDownloaderV3) sleep() {
-	for g.rate != nil && g.rate.Remaining <= 0 {
+	for g.rate != nil && g.rate.Remaining <= GithubLimitRateRemaining {
 		timer := time.NewTimer(time.Until(g.rate.Reset.Time))
 		select {
 		case <-g.ctx.Done():
@ -124,13 +126,22 @@ func (g *GithubDownloaderV3) sleep() {
 		case <-timer.C:
 		}

-		rates, _, err := g.client.RateLimits(g.ctx)
+		err := g.RefreshRate()
 		if err != nil {
 			log.Error("g.client.RateLimits: %s", err)
 		}
+	}
+}

-		g.rate = rates.GetCore()
+// RefreshRate update the current rate (doesn't count in rate limit)
+func (g *GithubDownloaderV3) RefreshRate() error {
+	rates, _, err := g.client.RateLimits(g.ctx)
+	if err != nil {
+		return err
 	}
+
+	g.rate = rates.GetCore()
+	return nil
 }

 // GetRepoInfo returns a repository information
--- a/modules/migrations/github_test.go
+++ b/modules/migrations/github_test.go
@ -6,6 +6,7 @@
 package migrations

 import (
+	"os"
 	"testing"
 	"time"

@ -62,7 +63,11 @@ func assertLabelEqual(t *testing.T, name, color, description string, label *base
 }

 func TestGitHubDownloadRepo(t *testing.T) {
-	downloader := NewGithubDownloaderV3("", "", "go-gitea", "test_repo")
+	GithubLimitRateRemaining = 3 //Wait at 3 remaining since we could have 3 CI in //
+	downloader := NewGithubDownloaderV3(os.Getenv("GITHUB_READ_TOKEN"), "", "go-gitea", "test_repo")
+	err := downloader.RefreshRate()
+	assert.NoError(t, err)
+
 	repo, err := downloader.GetRepoInfo()
 	assert.NoError(t, err)
 	assert.EqualValues(t, &base.Repository{