Add a new table issue_index to store the max issue index so that issue could be deleted with no duplicated index (#15599)
* Add a new table issue_index to store the max issue index so that issue could be deleted with no duplicated index * Fix pull index * Add tests for concurrent creating issues * Fix lint * Fix tests * Fix postgres test * Add test for migration v180 * Rename wrong test file name Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv>mj-v1.18.3
Lunny Xiao
3 years ago
committed by
GitHub
parent
a005265718
commit
0393a57511
@ -0,0 +1,24 @@
|
||||
-
|
||||
group_id: 1
|
||||
max_index: 5
|
||||
-
|
||||
group_id: 2
|
||||
max_index: 2
|
||||
-
|
||||
group_id: 3
|
||||
max_index: 2
|
||||
-
|
||||
group_id: 10
|
||||
max_index: 1
|
||||
-
|
||||
group_id: 48
|
||||
max_index: 1
|
||||
-
|
||||
group_id: 42
|
||||
max_index: 1
|
||||
-
|
||||
group_id: 50
|
||||
max_index: 1
|
||||
-
|
||||
group_id: 51
|
||||
max_index: 1
|
||||
@ -0,0 +1,113 @@
|
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package models
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
)
|
||||
|
||||
// ResourceIndex represents a resource index which could be used as issue/release and others
|
||||
// We can create different tables i.e. issue_index, release_index and etc.
|
||||
type ResourceIndex struct {
|
||||
GroupID int64 `xorm:"unique"`
|
||||
MaxIndex int64 `xorm:"index"`
|
||||
}
|
||||
|
||||
// IssueIndex represents the issue index table
|
||||
type IssueIndex ResourceIndex
|
||||
|
||||
// upsertResourceIndex the function will not return until it acquires the lock or receives an error.
|
||||
func upsertResourceIndex(e Engine, tableName string, groupID int64) (err error) {
|
||||
// An atomic UPSERT operation (INSERT/UPDATE) is the only operation
|
||||
// that ensures that the key is actually locked.
|
||||
switch {
|
||||
case setting.Database.UseSQLite3 || setting.Database.UsePostgreSQL:
|
||||
_, err = e.Exec(fmt.Sprintf("INSERT INTO %s (group_id, max_index) "+
|
||||
"VALUES (?,1) ON CONFLICT (group_id) DO UPDATE SET max_index = %s.max_index+1",
|
||||
tableName, tableName), groupID)
|
||||
case setting.Database.UseMySQL:
|
||||
_, err = e.Exec(fmt.Sprintf("INSERT INTO %s (group_id, max_index) "+
|
||||
"VALUES (?,1) ON DUPLICATE KEY UPDATE max_index = max_index+1", tableName),
|
||||
groupID)
|
||||
case setting.Database.UseMSSQL:
|
||||
// https://weblogs.sqlteam.com/dang/2009/01/31/upsert-race-condition-with-merge/
|
||||
_, err = e.Exec(fmt.Sprintf("MERGE %s WITH (HOLDLOCK) as target "+
|
||||
"USING (SELECT ? AS group_id) AS src "+
|
||||
"ON src.group_id = target.group_id "+
|
||||
"WHEN MATCHED THEN UPDATE SET target.max_index = target.max_index+1 "+
|
||||
"WHEN NOT MATCHED THEN INSERT (group_id, max_index) "+
|
||||
"VALUES (src.group_id, 1);", tableName),
|
||||
groupID)
|
||||
default:
|
||||
return fmt.Errorf("database type not supported")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
// ErrResouceOutdated represents an error when request resource outdated
|
||||
ErrResouceOutdated = errors.New("resource outdated")
|
||||
// ErrGetResourceIndexFailed represents an error when resource index retries 3 times
|
||||
ErrGetResourceIndexFailed = errors.New("get resource index failed")
|
||||
)
|
||||
|
||||
const (
|
||||
maxDupIndexAttempts = 3
|
||||
)
|
||||
|
||||
// GetNextResourceIndex retried 3 times to generate a resource index
|
||||
func GetNextResourceIndex(tableName string, groupID int64) (int64, error) {
|
||||
for i := 0; i < maxDupIndexAttempts; i++ {
|
||||
idx, err := getNextResourceIndex(tableName, groupID)
|
||||
if err == ErrResouceOutdated {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return idx, nil
|
||||
}
|
||||
return 0, ErrGetResourceIndexFailed
|
||||
}
|
||||
|
||||
// deleteResouceIndex delete resource index
|
||||
func deleteResouceIndex(e Engine, tableName string, groupID int64) error {
|
||||
_, err := e.Exec(fmt.Sprintf("DELETE FROM %s WHERE group_id=?", tableName), groupID)
|
||||
return err
|
||||
}
|
||||
|
||||
// getNextResourceIndex return the next index
|
||||
func getNextResourceIndex(tableName string, groupID int64) (int64, error) {
|
||||
sess := x.NewSession()
|
||||
defer sess.Close()
|
||||
if err := sess.Begin(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var preIdx int64
|
||||
_, err := sess.SQL(fmt.Sprintf("SELECT max_index FROM %s WHERE group_id = ?", tableName), groupID).Get(&preIdx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if err := upsertResourceIndex(sess, tableName, groupID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
var curIdx int64
|
||||
has, err := sess.SQL(fmt.Sprintf("SELECT max_index FROM %s WHERE group_id = ? AND max_index=?", tableName), groupID, preIdx+1).Get(&curIdx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if !has {
|
||||
return 0, ErrResouceOutdated
|
||||
}
|
||||
if err := sess.Commit(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return curIdx, nil
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestResourceIndex(t *testing.T) {
|
||||
assert.NoError(t, PrepareTestDatabase())
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 100; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
testInsertIssue(t, fmt.Sprintf("issue %d", i+1), "my issue", 0)
|
||||
wg.Done()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"xorm.io/xorm"
|
||||
)
|
||||
|
||||
func addIssueResourceIndexTable(x *xorm.Engine) error {
|
||||
type ResourceIndex struct {
|
||||
GroupID int64 `xorm:"index unique(s)"`
|
||||
MaxIndex int64 `xorm:"index unique(s)"`
|
||||
}
|
||||
|
||||
sess := x.NewSession()
|
||||
defer sess.Close()
|
||||
|
||||
if err := sess.Begin(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := sess.Table("issue_index").Sync2(new(ResourceIndex)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove data we're goint to rebuild
|
||||
if _, err := sess.Table("issue_index").Where("1=1").Delete(&ResourceIndex{}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create current data for all repositories with issues and PRs
|
||||
if _, err := sess.Exec("INSERT INTO issue_index (group_id, max_index) " +
|
||||
"SELECT max_data.repo_id, max_data.max_index " +
|
||||
"FROM ( SELECT issue.repo_id AS repo_id, max(issue.`index`) AS max_index " +
|
||||
"FROM issue GROUP BY issue.repo_id) AS max_data"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return sess.Commit()
|
||||
}
|
||||
@ -0,0 +1,59 @@
|
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func Test_addIssueResourceIndexTable(t *testing.T) {
|
||||
// Create the models used in the migration
|
||||
type Issue struct {
|
||||
ID int64 `xorm:"pk autoincr"`
|
||||
RepoID int64 `xorm:"UNIQUE(s)"`
|
||||
Index int64 `xorm:"UNIQUE(s)"`
|
||||
}
|
||||
|
||||
// Prepare and load the testing database
|
||||
x, deferable := prepareTestEnv(t, 0, new(Issue))
|
||||
if x == nil || t.Failed() {
|
||||
defer deferable()
|
||||
return
|
||||
}
|
||||
defer deferable()
|
||||
|
||||
// Run the migration
|
||||
if err := addIssueResourceIndexTable(x); err != nil {
|
||||
assert.NoError(t, err)
|
||||
return
|
||||
}
|
||||
|
||||
type ResourceIndex struct {
|
||||
GroupID int64 `xorm:"index unique(s)"`
|
||||
MaxIndex int64 `xorm:"index unique(s)"`
|
||||
}
|
||||
|
||||
var start = 0
|
||||
const batchSize = 1000
|
||||
for {
|
||||
var indexes = make([]ResourceIndex, 0, batchSize)
|
||||
err := x.Table("issue_index").Limit(batchSize, start).Find(&indexes)
|
||||
assert.NoError(t, err)
|
||||
|
||||
for _, idx := range indexes {
|
||||
var maxIndex int
|
||||
has, err := x.SQL("SELECT max(`index`) FROM issue WHERE repo_id = ?", idx.GroupID).Get(&maxIndex)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, has)
|
||||
assert.EqualValues(t, maxIndex, idx.MaxIndex)
|
||||
}
|
||||
if len(indexes) < batchSize {
|
||||
break
|
||||
}
|
||||
start += len(indexes)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue