type GoRoutineMap interface {
	// Run adds operation name to the list of running operations and spawns a
	// new go routine to execute the operation.
	// If an operation with the same operation name already exists, an
	// AlreadyExists or ExponentialBackoff error is returned.
	// Once the operation is complete, the go routine is terminated and the
	// operation name is removed from the list of executing operations allowing
	// a new operation to be started with the same operation name without error.
	Run(operationName string, operationFunc func() error) error
 
	// Wait blocks until operations map is empty. This is typically
	// necessary during tests - the test should wait until all operations finish
	// and evaluate results after that.
	Wait()
 
	// WaitForCompletion blocks until either all operations have successfully completed
	// or have failed but are not pending. The test should wait until operations are either
	// complete or have failed.
	WaitForCompletion()
 
	IsOperationPending(operationName string) bool
}

// goRoutineMap结构体实现GoRoutineMap接口，
type goRoutineMap struct {
	// 用于记录goRoutineMap维护协程的状态
	operations                map[string]operation
	// 发生错误时是否指数级补偿
	exponentialBackOffOnError bool
	// 用在多个 Goroutine 等待，一个 Goroutine 通知（事件发生）的场景
	cond                      *sync.Cond
	lock                      sync.RWMutex
}
 
// operation结构体对象维护单个goroutine的状态。
type operation struct {
	// 是否操作挂起
	operationPending bool
	// 单个goroutine执行逻辑报错时，实现以指数退避方式
	expBackoff       exponentialbackoff.ExponentialBackoff
}

// ExponentialBackoff contains the last occurrence of an error and the duration
// that retries are not permitted.
type ExponentialBackoff struct {
	lastError           error
	lastErrorTime       time.Time
	durationBeforeRetry time.Duration
}

// NewGoRoutineMap returns a new instance of GoRoutineMap.
func NewGoRoutineMap(exponentialBackOffOnError bool) GoRoutineMap {
	g := &goRoutineMap{
		operations:                make(map[string]operation),
		exponentialBackOffOnError: exponentialBackOffOnError,
	}
 
	g.cond = sync.NewCond(&g.lock)
	return g
}

// Run函数是外部函数，是goRoutineMap核心方法，其能够防止创建具有相同名称的多个goroutine，并使用协程的方式运行函数逻辑
// 如果函数成功执行，则退出该协程；如果函数执行报错，在指数退避的时间内禁止再次执行该函数逻辑。
func (grm *goRoutineMap) Run(
	operationName string,
	operationFunc func() error) error {
	grm.lock.Lock()
	defer grm.lock.Unlock()
 
	// 判断grm.operations这个map中是否存在具有operationName名称的协程
	existingOp, exists := grm.operations[operationName]
	if exists {
		// 如果grm.operations这个map中已经存在operationName名称的协程，并且existingOp.operationPending==true，说明grm.operations中operationName名称这个协程正在执行函数逻辑，在这期间又有一个同名的
		// operationName希望加入grm.operations这个map，此时加入map失败并报AlreadyExistsError错误
		if existingOp.operationPending {
			return NewAlreadyExistsError(operationName)
		}
 
		// 到这步说明名称为operationName名称的协程执行函数逻辑失败，此时判断此协程最后一次失败时间 + 指数退避的时间 >= 当前时间，如果不符合条件的话禁止执行该协程函数逻辑。
		if err := existingOp.expBackoff.SafeToRetry(operationName); err != nil {
			return err
		}
	}
 
	// 如果grm.operations这个map中不存在operationName名称的协程 或者 此协程最后一次失败时间 + 指数退避的时间 < 当前时间，则在grm.operations这个map中重新维护此协程（注意，operationPending=true）
	grm.operations[operationName] = operation{
		operationPending: true,
		expBackoff:       existingOp.expBackoff,
	}
 
	// 以协程方式执行函数逻辑operationFunc()
	go func() (err error) {
		// 捕获崩溃并记录错误，默认不传参的话，在程序发送崩溃时，在控制台打印一下崩溃日志后再崩溃，方便技术人员排查程序错误。
		defer k8sRuntime.HandleCrash()
 
		// 如果执行operationFunc()函数逻辑不报错或者grm.exponentialBackOffOnError=false的话，将从grm.operations这个map中移除此operationName名称协程；
		// 如果执行operationFunc()函数逻辑报错并且grm.exponentialBackOffOnError=true，则将产生指数级补偿，到达补偿时间后才能再调用此operationName名称协程的函数逻辑
		// Handle completion of and error, if any, from operationFunc()
		defer grm.operationComplete(operationName, &err)
		// 处理operationFunc()函数发生的panic错误，以便defer grm.operationComplete(operationName, &err)能执行
		// Handle panic, if any, from operationFunc()
		defer k8sRuntime.RecoverFromPanic(&err)
		return operationFunc()
	}()
 
	return nil
}

// SafeToRetry returns an error if the durationBeforeRetry period for the given
// lastErrorTime has not yet expired. Otherwise it returns nil.
func (expBackoff *ExponentialBackoff) SafeToRetry(operationName string) error {
	if time.Since(expBackoff.lastErrorTime) <= expBackoff.durationBeforeRetry {
		return NewExponentialBackoffError(operationName, *expBackoff)
	}
 
	return nil
}

// operationComplete是一个内部函数，用于处理在goRoutineMap中已经运行完函数逻辑的协程
// 如果执行operationFunc()函数逻辑不报错或者grm.exponentialBackOffOnError=false的话，将从grm.operations这个map中移除此operationName名称协程；
// 如果执行operationFunc()函数逻辑报错并且grm.exponentialBackOffOnError=true，则将产生指数级补偿，到达补偿时间后才能再调用此operationName名称协程的函数逻辑
// operationComplete handles the completion of a goroutine run in the
// goRoutineMap.
func (grm *goRoutineMap) operationComplete(
	operationName string, err *error) {
	// Defer operations are executed in Last-In is First-Out order. In this case
	// the lock is acquired first when operationCompletes begins, and is
	// released when the method finishes, after the lock is released cond is
	// signaled to wake waiting goroutine.
	defer grm.cond.Signal()
	grm.lock.Lock()
	defer grm.lock.Unlock()
 
	if *err == nil || !grm.exponentialBackOffOnError {
		// 函数逻辑执行完成无错误或已禁用错误指数级补偿，将从grm.operations这个map中移除此operationName名称协程；
		// Operation completed without error, or exponentialBackOffOnError disabled
		delete(grm.operations, operationName)
		if *err != nil {
			// Log error
			klog.Errorf("operation for %q failed with: %v",
				operationName,
				*err)
		}
	} else {
		//  函数逻辑执行完成有错误则将产生指数级补偿，到达补偿时间后才能再调用此operationName名称协程的函数逻辑（注意，指数补充的协程，operationPending=false）
		// Operation completed with error and exponentialBackOffOnError Enabled
		existingOp := grm.operations[operationName]
		existingOp.expBackoff.Update(err)
		existingOp.operationPending = false
		grm.operations[operationName] = existingOp
 
		// Log error
		klog.Errorf("%v",
			existingOp.expBackoff.GenerateNoRetriesPermittedMsg(operationName))
	}
}

func (expBackoff *ExponentialBackoff) Update(err *error) {
	if expBackoff.durationBeforeRetry == 0 {
		expBackoff.durationBeforeRetry = initialDurationBeforeRetry
	} else {
		expBackoff.durationBeforeRetry = 2 * expBackoff.durationBeforeRetry
		if expBackoff.durationBeforeRetry > maxDurationBeforeRetry {
			expBackoff.durationBeforeRetry = maxDurationBeforeRetry
		}
	}
 
	expBackoff.lastError = *err
	expBackoff.lastErrorTime = time.Now()
}