diff --git a/models/consistency.go b/models/consistency.go
index 3a2208014..bed94d939 100644
--- a/models/consistency.go
+++ b/models/consistency.go
@@ -319,6 +319,64 @@ func FixCommentTypeLabelWithEmptyLabel() (int64, error) {
 	return x.Where(builder.Eq{"type": CommentTypeLabel, "label_id": 0}).Delete(new(Comment))
 }
 
+// CountCommentTypeLabelWithOutsideLabels count label comments with outside label
+func CountCommentTypeLabelWithOutsideLabels() (int64, error) {
+	return x.Where("comment.type = ? AND (issue.repo_id != label.repo_id OR (label.repo_id = 0 AND repository.owner_id != label.org_id))", CommentTypeLabel).
+		Table("comment").
+		Join("inner", "label", "label.id = comment.label_id").
+		Join("inner", "issue", "issue.id = comment.issue_id ").
+		Join("inner", "repository", "issue.repo_id = repository.id").
+		Count(new(Comment))
+}
+
+// FixCommentTypeLabelWithOutsideLabels count label comments with outside label
+func FixCommentTypeLabelWithOutsideLabels() (int64, error) {
+	res, err := x.Exec(`DELETE FROM comment WHERE comment.id IN (
+		SELECT il_too.id FROM (
+			SELECT com.id
+				FROM comment AS com
+					INNER JOIN label ON com.label_id = label.id
+					INNER JOIN issue on issue.id = com.issue_id
+				WHERE
+					com.type = ? AND (issue.repo_id != label.repo_id OR (label.repo_id = 0 AND label.org_id != repo.owner_id))
+	) AS il_too)`, CommentTypeLabel)
+	if err != nil {
+		return 0, err
+	}
+
+	return res.RowsAffected()
+}
+
+// CountIssueLabelWithOutsideLabels count label comments with outside label
+func CountIssueLabelWithOutsideLabels() (int64, error) {
+	return x.Where(builder.Expr("issue.repo_id != label.repo_id OR (label.repo_id = 0 AND repository.owner_id != label.org_id)")).
+		Table("issue_label").
+		Join("inner", "label", "issue_label.id = label.id ").
+		Join("inner", "issue", "issue.id = issue_label.issue_id ").
+		Join("inner", "repository", "issue.repo_id = repository.id").
+		Count(new(IssueLabel))
+}
+
+// FixIssueLabelWithOutsideLabels fix label comments with outside label
+func FixIssueLabelWithOutsideLabels() (int64, error) {
+	res, err := x.Exec(`DELETE FROM issue_label WHERE issue_label.id IN (
+		SELECT il_too.id FROM (
+			SELECT il_too_too.id
+				FROM issue_label AS il_too_too
+					INNER JOIN label ON il_too_too.id = label.id
+					INNER JOIN issue on issue.id = il_too_too.issue_id
+					INNER JOIN repository on repository.id = issue.repo_id
+				WHERE
+					issue.repo_id != label.repo_id OR (label.repo_id = 0 AND label.org_id != repository.owner_id)
+	) AS il_too )`)
+
+	if err != nil {
+		return 0, err
+	}
+
+	return res.RowsAffected()
+}
+
 // CountBadSequences looks for broken sequences from recreate-table mistakes
 func CountBadSequences() (int64, error) {
 	if !setting.Database.UsePostgreSQL {
diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go
index f46d151b7..3427b5dec 100644
--- a/models/migrations/migrations.go
+++ b/models/migrations/migrations.go
@@ -298,6 +298,8 @@ var migrations = []Migration{
 	NewMigration("create repo transfer table", addRepoTransfer),
 	// v175 -> v176
 	NewMigration("Fix Postgres ID Sequences broken by recreate-table", fixPostgresIDSequences),
+	// v176 -> v177
+	NewMigration("Remove invalid labels from comments", removeInvalidLabels),
 }
 
 // GetCurrentDBVersion returns the current db version
diff --git a/models/migrations/v176.go b/models/migrations/v176.go
new file mode 100644
index 000000000..ff6587508
--- /dev/null
+++ b/models/migrations/v176.go
@@ -0,0 +1,74 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"xorm.io/xorm"
+)
+
+func removeInvalidLabels(x *xorm.Engine) error {
+	type Comment struct {
+		ID      int64 `xorm:"pk autoincr"`
+		Type    int   `xorm:"INDEX"`
+		IssueID int64 `xorm:"INDEX"`
+		LabelID int64
+	}
+
+	type Issue struct {
+		ID     int64 `xorm:"pk autoincr"`
+		RepoID int64 `xorm:"INDEX UNIQUE(repo_index)"`
+		Index  int64 `xorm:"UNIQUE(repo_index)"` // Index in one repository.
+	}
+
+	type Repository struct {
+		ID        int64  `xorm:"pk autoincr"`
+		OwnerID   int64  `xorm:"UNIQUE(s) index"`
+		LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"`
+	}
+
+	type Label struct {
+		ID     int64 `xorm:"pk autoincr"`
+		RepoID int64 `xorm:"INDEX"`
+		OrgID  int64 `xorm:"INDEX"`
+	}
+
+	type IssueLabel struct {
+		ID      int64 `xorm:"pk autoincr"`
+		IssueID int64 `xorm:"UNIQUE(s)"`
+		LabelID int64 `xorm:"UNIQUE(s)"`
+	}
+
+	if err := x.Sync2(new(Comment), new(Issue), new(Repository), new(Label), new(IssueLabel)); err != nil {
+		return err
+	}
+
+	if _, err := x.Exec(`DELETE FROM issue_label WHERE issue_label.id IN (
+		SELECT il_too.id FROM (
+			SELECT il_too_too.id
+				FROM issue_label AS il_too_too
+					INNER JOIN label ON il_too_too.id = label.id
+					INNER JOIN issue on issue.id = il_too_too.issue_id
+					INNER JOIN repository on repository.id = issue.repo_id
+				WHERE
+					issue.repo_id != label.repo_id OR (label.repo_id = 0 AND label.org_id != repository.owner_id)
+	) AS il_too )`); err != nil {
+		return err
+	}
+
+	if _, err := x.Exec(`DELETE FROM comment WHERE comment.id IN (
+		SELECT il_too.id FROM (
+			SELECT com.id
+				FROM comment AS com
+					INNER JOIN label ON com.label_id = label.id
+					INNER JOIN issue on issue.id = com.issue_id
+					INNER JOIN repository on repository.id = issue.repo_id
+				WHERE
+					com.type = ? AND (issue.repo_id != label.repo_id OR (label.repo_id = 0 AND label.org_id != repository.owner_id))
+	) AS il_too)`, 7); err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/modules/doctor/dbconsistency.go b/modules/doctor/dbconsistency.go
index 1e3c7258e..83d3dc5fe 100644
--- a/modules/doctor/dbconsistency.go
+++ b/modules/doctor/dbconsistency.go
@@ -130,6 +130,45 @@ func checkDBConsistency(logger log.Logger, autofix bool) error {
 			logger.Warn("%d label comments with empty labels", count)
 		}
 	}
+
+	// find label comments with labels from outside the repository
+	count, err = models.CountCommentTypeLabelWithOutsideLabels()
+	if err != nil {
+		logger.Critical("Error: %v whilst counting label comments with outside labels", err)
+		return err
+	}
+	if count > 0 {
+		if autofix {
+			updatedCount, err := models.FixCommentTypeLabelWithOutsideLabels()
+			if err != nil {
+				logger.Critical("Error: %v whilst removing label comments with outside labels", err)
+				return err
+			}
+			log.Info("%d label comments with outside labels removed", updatedCount)
+		} else {
+			log.Warn("%d label comments with outside labels", count)
+		}
+	}
+
+	// find issue_label with labels from outside the repository
+	count, err = models.CountIssueLabelWithOutsideLabels()
+	if err != nil {
+		logger.Critical("Error: %v whilst counting issue_labels from outside the repository or organisation", err)
+		return err
+	}
+	if count > 0 {
+		if autofix {
+			updatedCount, err := models.FixIssueLabelWithOutsideLabels()
+			if err != nil {
+				logger.Critical("Error: %v whilst removing issue_labels from outside the repository or organisation", err)
+				return err
+			}
+			logger.Info("%d issue_labels from outside the repository or organisation removed", updatedCount)
+		} else {
+			logger.Warn("%d issue_labels from outside the repository or organisation", count)
+		}
+	}
+
 	// TODO: function to recalc all counters
 
 	if setting.Database.UsePostgreSQL {