mono
2f7a491666
Source-only publication of reviewed commit 64e96a7f423673dc29ad56485be9fc79bce8af1c.
Passed in isolation: focused frontend/backend tests, race checks, TypeScript typecheck, lint, Go compilation, Flowgram build, and deterministic SDK build/tests.
Not accepted as a production release: Next.js export/native assembly remains blocked by environment memory limits; browser interaction and MySQL staging acceptance remain unverified. No production server, database/schema, runtime configuration, deployment, release tag, or workflow execution was performed.
Expected source tree: 0a4a7528e9.
451 lines
18 KiB
Go
451 lines
18 KiB
Go
package services
|
|
|
|
import (
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"agent-desk/internal/models"
|
|
"agent-desk/internal/pkg/config"
|
|
"agent-desk/internal/pkg/dto/response"
|
|
"agent-desk/internal/pkg/enums"
|
|
"agent-desk/internal/pkg/openidentity"
|
|
"agent-desk/internal/repositories"
|
|
|
|
"github.com/gin-gonic/gin"
|
|
"github.com/glebarez/sqlite"
|
|
"github.com/golang-jwt/jwt/v5"
|
|
"github.com/mlogclub/simple/sqls"
|
|
"gorm.io/gorm"
|
|
"gorm.io/gorm/logger"
|
|
"gorm.io/gorm/schema"
|
|
)
|
|
|
|
func setupIdentitySecurityDB(t *testing.T) (*gorm.DB, *models.Channel) {
|
|
t.Helper()
|
|
db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "identity.db")), &gorm.Config{
|
|
NamingStrategy: schema.NamingStrategy{TablePrefix: "t_", SingularTable: true},
|
|
Logger: logger.Default.LogMode(logger.Silent),
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.AutoMigrate(&models.Customer{}, &models.CustomerIdentity{}, &models.Channel{}, &models.Conversation{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
previousDB, previousConfig := sqls.DB(), config.GetCurrent()
|
|
sqls.SetDB(db)
|
|
config.SetCurrent(&config.Config{CustomerSession: config.CustomerSessionConfig{Secret: "test-only-customer-session-key", TTLMinutes: 120}})
|
|
t.Cleanup(func() {
|
|
sqls.SetDB(previousDB)
|
|
config.SetCurrent(previousConfig)
|
|
sqlDB, _ := db.DB()
|
|
_ = sqlDB.Close()
|
|
})
|
|
channel := &models.Channel{ChannelID: "channel-a", ChannelType: enums.ChannelTypeWeb, Status: enums.StatusOk, ConfigJSON: "{}"}
|
|
if err := db.Create(channel).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return db, channel
|
|
}
|
|
|
|
func scopedTestExternal(channel *models.Channel, issuer, id string) openidentity.ExternalUser {
|
|
source := enums.ExternalSourceUser
|
|
if issuer == "guest" {
|
|
source = enums.ExternalSourceGuest
|
|
}
|
|
return openidentity.ExternalUser{ExternalSource: source, ExternalID: id, IdentityScope: openidentity.IdentityScope(channel.ChannelID, issuer), ChannelID: channel.ID}
|
|
}
|
|
|
|
func verifySecurityCustomerSession(t *testing.T, channel *models.Channel, token string) *CustomerSessionVerifyResult {
|
|
t.Helper()
|
|
ctx, _ := gin.CreateTestContext(httptest.NewRecorder())
|
|
ctx.Request = httptest.NewRequest("GET", "/api/ws/open", nil)
|
|
ctx.Request.Header.Set("Authorization", "Bearer "+token)
|
|
result, err := CustomerSessionService.VerifyRequest(ctx, channel)
|
|
if err != nil {
|
|
t.Fatalf("verify customer credential: %v", err)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func TestCustomerIdentitySeparatesTrustIssuerAndChannel(t *testing.T) {
|
|
db, channel := setupIdentitySecurityDB(t)
|
|
otherChannel := &models.Channel{ChannelID: "channel-b", ChannelType: enums.ChannelTypeWeb, Status: enums.StatusOk, ConfigJSON: "{}"}
|
|
if err := db.Create(otherChannel).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
customerIDs := map[int64]bool{}
|
|
for _, input := range []struct {
|
|
channel *models.Channel
|
|
issuer string
|
|
}{{channel, "hmac"}, {channel, "jwt"}, {channel, "guest"}, {otherChannel, "hmac"}} {
|
|
exchanged, err := CustomerSessionService.Exchange(input.channel, scopedTestExternal(input.channel, input.issuer, "same-id"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if customerIDs[exchanged.Customer.ID] {
|
|
t.Fatal("independent identity scopes were merged")
|
|
}
|
|
customerIDs[exchanged.Customer.ID] = true
|
|
result := verifySecurityCustomerSession(t, input.channel, exchanged.CustomerSessionToken)
|
|
if !CustomerSessionService.VerifyConnection(result) {
|
|
t.Fatal("fresh scoped connection rejected")
|
|
}
|
|
wrongChannel := &models.Conversation{CustomerID: result.CustomerID, ChannelID: otherChannel.ID}
|
|
if input.channel.ID != otherChannel.ID && ConversationService.IsCustomerConversationOwner(wrongChannel, *result.ExternalUser) {
|
|
t.Fatal("customer acquired a different channel's conversation")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGuestBootstrapCannotReclaimKnownAnonymousIdentity(t *testing.T) {
|
|
_, channel := setupIdentitySecurityDB(t)
|
|
first, err := CustomerSessionService.Exchange(channel, scopedTestExternal(channel, "guest", "caller-chosen-id"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stolenID := strings.TrimPrefix(first.IdentityKey, "guest:")
|
|
if len(stolenID) != 64 || stolenID == "caller-chosen-id" {
|
|
t.Fatal("guest identity was not server-generated")
|
|
}
|
|
second, err := CustomerSessionService.Exchange(channel, scopedTestExternal(channel, "guest", stolenID))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if first.Customer.ID == second.Customer.ID || first.IdentityKey == second.IdentityKey {
|
|
t.Fatal("public guest bootstrap reclaimed another guest identity")
|
|
}
|
|
original := verifySecurityCustomerSession(t, channel, first.CustomerSessionToken)
|
|
if original.CustomerID != first.Customer.ID {
|
|
t.Fatal("signed guest credential did not retain its owner")
|
|
}
|
|
}
|
|
|
|
func TestCustomerIdentityDuplicatesFailClosedWithoutCreatingAnotherCustomer(t *testing.T) {
|
|
db, channel := setupIdentitySecurityDB(t)
|
|
external := scopedTestExternal(channel, "hmac", "duplicate")
|
|
for i := 0; i < 2; i++ {
|
|
customer := &models.Customer{Status: enums.StatusOk}
|
|
if err := db.Create(customer).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
identity := &models.CustomerIdentity{CustomerID: customer.ID, IdentityScope: external.IdentityScope, ExternalSource: external.ExternalSource, ExternalID: external.ExternalID, Status: enums.StatusOk}
|
|
if err := db.Create(identity).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := CustomerSessionService.Exchange(channel, external); err == nil {
|
|
t.Fatal("ambiguous identity was accepted")
|
|
}
|
|
var count int64
|
|
if err := db.Model(&models.Customer{}).Count(&count).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if count != 2 {
|
|
t.Fatalf("ambiguous mapping created another customer: %d", count)
|
|
}
|
|
identity, err := repositories.CustomerIdentityRepository.GetUniqueBy(db, external.IdentityScope, external.ExternalSource, external.ExternalID)
|
|
if err == nil || identity != nil {
|
|
t.Fatal("duplicate lookup did not fail closed")
|
|
}
|
|
}
|
|
|
|
func TestCustomerSessionsRejectLegacyScopeAndLivePolicyChanges(t *testing.T) {
|
|
db, channel := setupIdentitySecurityDB(t)
|
|
exchanged, err := CustomerSessionService.Exchange(channel, scopedTestExternal(channel, "guest", "guest"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
current := verifySecurityCustomerSession(t, channel, exchanged.CustomerSessionToken)
|
|
claims, err := CustomerSessionService.verifyToken(exchanged.CustomerSessionToken)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
claims.IdentityScope = ""
|
|
legacy, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(config.Current().CustomerSession.Secret))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := CustomerSessionService.verifyToken(legacy); err == nil {
|
|
t.Fatal("unscoped legacy credential accepted")
|
|
}
|
|
if err := db.Model(channel).Update("config_json", `{"requireSignedIdentity":true}`).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if CustomerSessionService.VerifyConnection(current) {
|
|
t.Fatal("existing guest retained access after signed-only policy change")
|
|
}
|
|
channel.ConfigJSON = `{"requireSignedIdentity":true}`
|
|
if _, err := CustomerSessionService.Exchange(channel, scopedTestExternal(channel, "guest", "guest")); err == nil {
|
|
t.Fatal("signed-only channel accepted guest exchange")
|
|
}
|
|
}
|
|
|
|
func TestCustomerConnectionRejectsExpiryDisabledCustomerAndIdentityChanges(t *testing.T) {
|
|
for _, scenario := range []string{"expiry", "disabled-customer", "disabled-channel", "disabled-identity", "rebound-identity", "duplicate-identity"} {
|
|
t.Run(scenario, func(t *testing.T) {
|
|
db, channel := setupIdentitySecurityDB(t)
|
|
exchanged, err := CustomerSessionService.Exchange(channel, scopedTestExternal(channel, "hmac", "signed"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
current := verifySecurityCustomerSession(t, channel, exchanged.CustomerSessionToken)
|
|
switch scenario {
|
|
case "expiry":
|
|
current.ExpiresAt = time.Now()
|
|
case "disabled-customer":
|
|
err = db.Model(&models.Customer{}).Where("id = ?", current.CustomerID).Update("status", enums.StatusDisabled).Error
|
|
case "disabled-channel":
|
|
err = db.Model(channel).Update("status", enums.StatusDisabled).Error
|
|
case "disabled-identity":
|
|
err = db.Model(&models.CustomerIdentity{}).Where("customer_id = ?", current.CustomerID).Update("status", enums.StatusDisabled).Error
|
|
case "rebound-identity":
|
|
err = db.Model(&models.CustomerIdentity{}).Where("customer_id = ?", current.CustomerID).Update("customer_id", current.CustomerID+100).Error
|
|
case "duplicate-identity":
|
|
err = db.Create(&models.CustomerIdentity{CustomerID: current.CustomerID + 100, ExternalSource: current.ExternalUser.ExternalSource, ExternalID: current.ExternalUser.ExternalID, IdentityScope: current.ExternalUser.IdentityScope, Status: enums.StatusOk}).Error
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if CustomerSessionService.VerifyConnection(current) {
|
|
t.Fatal("invalidated customer connection retained authority")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCustomerRealtimeTopicsAndDeliveryRejectCrossScopeCollisions(t *testing.T) {
|
|
db, channel := setupIdentitySecurityDB(t)
|
|
ws := newWsService()
|
|
signed, err := CustomerSessionService.Exchange(channel, scopedTestExternal(channel, "hmac", "known-id"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
guest, err := CustomerSessionService.Exchange(channel, scopedTestExternal(channel, "guest", "known-id"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
signedAuth := verifySecurityCustomerSession(t, channel, signed.CustomerSessionToken)
|
|
guestAuth := verifySecurityCustomerSession(t, channel, guest.CustomerSessionToken)
|
|
conversation := &models.Conversation{CustomerID: signed.Customer.ID, ChannelID: channel.ID}
|
|
if err := db.Create(conversation).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
owner := &ClientSession{ID: "owner", Role: realtimeRoleUser, External: signedAuth.ExternalUser, CustomerSession: signedAuth, Topics: map[string]struct{}{}, Send: make(chan []byte, 4)}
|
|
attacker := &ClientSession{ID: "guest", Role: realtimeRoleUser, External: guestAuth.ExternalUser, CustomerSession: guestAuth, Topics: map[string]struct{}{}, Send: make(chan []byte, 4)}
|
|
if ws.customerTopic(signed.Customer.ID, channel.ID) == ws.customerTopic(guest.Customer.ID, channel.ID) {
|
|
t.Fatal("trust namespaces collided")
|
|
}
|
|
if ws.customerTopic(signed.Customer.ID, channel.ID) == ws.customerTopic(signed.Customer.ID, channel.ID+1) {
|
|
t.Fatal("channel namespaces collided")
|
|
}
|
|
ws.manager.Register(owner, ws.defaultTopics(owner))
|
|
// Defense in depth: even an accidentally stale/wrong topic registration may
|
|
// not grant the guest authorization to the signed customer's conversation.
|
|
ws.manager.Register(attacker, ws.routeConversationTopics(conversation))
|
|
ws.PublishAgentActivity(conversation, "step", "", "running")
|
|
select {
|
|
case <-owner.Send:
|
|
default:
|
|
t.Fatal("owner did not receive its event")
|
|
}
|
|
select {
|
|
case <-attacker.Send:
|
|
t.Fatal("guest received another customer's event")
|
|
default:
|
|
}
|
|
if ws.canSubscribeConversation(attacker, conversation.ID) {
|
|
t.Fatal("explicit foreign conversation subscription allowed")
|
|
}
|
|
}
|
|
|
|
func TestScopedIdentityUniqueClaimPreservesLegacyRows(t *testing.T) {
|
|
db, channel := setupIdentitySecurityDB(t)
|
|
source := enums.ExternalSourceUser
|
|
// Dirty legacy mappings remain migratable because their unique claim is NULL.
|
|
for _, customerID := range []int64{100, 101} {
|
|
if err := db.Create(&models.CustomerIdentity{CustomerID: customerID, ExternalSource: source, ExternalID: "same", Status: enums.StatusOk}).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
scope := openidentity.IdentityScope(channel.ChannelID, "hmac")
|
|
first := &models.CustomerIdentity{CustomerID: 102, IdentityScope: scope, ScopedKey: repositories.ScopedCustomerIdentityKey(scope, source, "same"), ExternalSource: source, ExternalID: "same", Status: enums.StatusOk}
|
|
if err := db.Create(first).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
competing := &models.CustomerIdentity{CustomerID: 103, IdentityScope: scope, ScopedKey: repositories.ScopedCustomerIdentityKey(scope, source, "same"), ExternalSource: source, ExternalID: "same", Status: enums.StatusOk}
|
|
if err := db.Create(competing).Error; err == nil {
|
|
t.Fatal("database allowed a competing scoped identity claim")
|
|
}
|
|
otherScope := openidentity.IdentityScope(channel.ChannelID, "jwt")
|
|
if err := db.Create(&models.CustomerIdentity{CustomerID: 104, IdentityScope: otherScope, ScopedKey: repositories.ScopedCustomerIdentityKey(otherScope, source, "same"), ExternalSource: source, ExternalID: "same", Status: enums.StatusOk}).Error; err != nil {
|
|
t.Fatal("different issuer's claim was conflated")
|
|
}
|
|
found, err := repositories.CustomerIdentityRepository.GetUniqueBy(db, scope, source, "same")
|
|
if err != nil || found == nil || found.CustomerID != 102 {
|
|
t.Fatal("legacy mappings interfered with scoped identity")
|
|
}
|
|
}
|
|
|
|
func TestWebIdentityPolicySurvivesOlderClientEdits(t *testing.T) {
|
|
svc := newChannelService()
|
|
raw, err := svc.preserveWebIdentityPolicy(`{"title":"New title"}`, `{"requireSignedIdentity":true}`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cfg, err := svc.ParseWebChannelConfig(raw)
|
|
if err != nil || !cfg.RequireSignedIdentity {
|
|
t.Fatal("older client edit silently disabled signed-only policy")
|
|
}
|
|
raw, err = svc.preserveWebIdentityPolicy(`{"requireSignedIdentity":false}`, `{"requireSignedIdentity":true}`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cfg, err = svc.ParseWebChannelConfig(raw)
|
|
if err != nil || cfg.RequireSignedIdentity {
|
|
t.Fatal("explicit policy change was not honored")
|
|
}
|
|
for _, raw := range []string{`{"requireSignedIdentity":null}`, `{"requireSignedIdentity":"false"}`, `{"requireSignedIdentity":0}`} {
|
|
if _, err := svc.ParseWebChannelConfig(raw); err == nil {
|
|
t.Fatal("non-boolean authentication policy accepted")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCustomerSessionNearExpiryRefreshPreservesIdentity(t *testing.T) {
|
|
db, channel := setupIdentitySecurityDB(t)
|
|
exchanged, err := CustomerSessionService.Exchange(channel, scopedTestExternal(channel, "guest", "local-context"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
claims, err := CustomerSessionService.verifyToken(exchanged.CustomerSessionToken)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
originalKey, originalScope, originalCustomer := claims.IdentityKey, claims.IdentityScope, claims.CustomerID
|
|
claims.ExpiresAt = jwt.NewNumericDate(time.Now().Add(20 * time.Second))
|
|
nearExpiry, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(config.Current().CustomerSession.Secret))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
refreshed := verifySecurityCustomerSession(t, channel, nearExpiry)
|
|
if !refreshed.Refreshed || !refreshed.ExpiresAt.After(claims.ExpiresAt.Time) || refreshed.Token == nearExpiry {
|
|
t.Fatal("near-expiry credential was not refreshed")
|
|
}
|
|
refreshedClaims, err := CustomerSessionService.verifyToken(refreshed.Token)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if refreshedClaims.IdentityKey != originalKey || refreshedClaims.IdentityScope != originalScope || refreshedClaims.CustomerID != originalCustomer {
|
|
t.Fatal("refresh changed anonymous identity")
|
|
}
|
|
var customers int64
|
|
if err := db.Model(&models.Customer{}).Count(&customers).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if customers != 1 {
|
|
t.Fatal("refresh allocated another customer")
|
|
}
|
|
claims.ExpiresAt = jwt.NewNumericDate(time.Now().Add(-time.Second))
|
|
expired, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(config.Current().CustomerSession.Secret))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, _ := gin.CreateTestContext(httptest.NewRecorder())
|
|
ctx.Request = httptest.NewRequest("POST", "/api/customer/session_refresh", nil)
|
|
ctx.Request.Header.Set("Authorization", "Bearer "+expired)
|
|
if _, err := CustomerSessionService.VerifyRequest(ctx, channel); err == nil {
|
|
t.Fatal("expired anonymous credential was silently renewed")
|
|
}
|
|
}
|
|
|
|
func TestConcurrentCustomerSessionExchangesKeepOneScopedOwner(t *testing.T) {
|
|
db, channel := setupIdentitySecurityDB(t)
|
|
if err := db.Exec("PRAGMA journal_mode=WAL").Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pool, err := db.DB()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pool.SetMaxOpenConns(4)
|
|
external := scopedTestExternal(channel, "hmac", "concurrent-id")
|
|
var queries atomic.Int32
|
|
arrived := make(chan struct{}, 2)
|
|
release := make(chan struct{})
|
|
var releaseOnce sync.Once
|
|
releaseBoth := func() { releaseOnce.Do(func() { close(release) }) }
|
|
var exchanges sync.WaitGroup
|
|
defer func() { releaseBoth(); exchanges.Wait() }()
|
|
callbackName := "test:concurrent_scoped_lookup"
|
|
if err := db.Callback().Query().After("gorm:query").Register(callbackName, func(tx *gorm.DB) {
|
|
if tx.Statement.Table != "t_customer_identity" || tx.Error != nil || tx.RowsAffected != 0 {
|
|
return
|
|
}
|
|
if queries.Add(1) <= 2 {
|
|
arrived <- struct{}{}
|
|
<-release
|
|
}
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = db.Callback().Query().Remove(callbackName) })
|
|
type result struct {
|
|
value *response.CustomerSessionExchangeResponse
|
|
err error
|
|
}
|
|
results := make(chan result, 2)
|
|
exchanges.Add(2)
|
|
for i := 0; i < 2; i++ {
|
|
go func() {
|
|
defer exchanges.Done()
|
|
value, err := CustomerSessionService.Exchange(channel, external)
|
|
results <- result{value, err}
|
|
}()
|
|
}
|
|
for i := 0; i < 2; i++ {
|
|
select {
|
|
case <-arrived:
|
|
case <-time.After(2 * time.Second):
|
|
releaseBoth()
|
|
t.Fatal("concurrent transactions did not both reach initial identity lookup")
|
|
}
|
|
}
|
|
releaseBoth()
|
|
successes := 0
|
|
var owner int64
|
|
for i := 0; i < 2; i++ {
|
|
select {
|
|
case result := <-results:
|
|
if result.err == nil {
|
|
successes++
|
|
owner = result.value.Customer.ID
|
|
}
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("concurrent exchange did not finish")
|
|
}
|
|
}
|
|
if successes != 1 {
|
|
t.Fatalf("expected one transaction winner, got %d", successes)
|
|
}
|
|
var customers, identities int64
|
|
if err := db.Model(&models.Customer{}).Count(&customers).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.Model(&models.CustomerIdentity{}).Count(&identities).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if customers != 1 || identities != 1 {
|
|
t.Fatal("losing transaction left an orphan or competing customer mapping")
|
|
}
|
|
retry, err := CustomerSessionService.Exchange(channel, external)
|
|
if err != nil || retry.Customer.ID != owner {
|
|
t.Fatal("retry did not resolve the unique winning owner")
|
|
}
|
|
}
|