From 11a2ff48d6bd85333181aedb87d59d6ce4939c2a Mon Sep 17 00:00:00 2001 From: Simone Gotti Date: Mon, 8 Jul 2019 16:03:14 +0200 Subject: [PATCH] runservice: delete executor task early currently we are deleting the executor tasks only when all the run tasks log/archives were fetched. But it'll better to remove a single executor task when the task fetching is finished. This could also fix possible issues on k8s since we are scheduling tasks but the k8s scheduler may not schedule them if there aren't enough resources causing a scheduling deadlock since we won't remove finished pods because their related tasks are not removed and k8s cannot start new pods since it has no resources. --- internal/services/runservice/scheduler.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/internal/services/runservice/scheduler.go b/internal/services/runservice/scheduler.go index 23e9106..7b772db 100644 --- a/internal/services/runservice/scheduler.go +++ b/internal/services/runservice/scheduler.go @@ -1177,6 +1177,15 @@ func (s *Runservice) fetcher(ctx context.Context) error { s.fetchTaskLogs(ctx, r.ID, rt) s.fetchTaskArchives(ctx, r.ID, rt) + + // if the fetching is finished we can remove the executor tasks. We cannot + // remove it before since it contains the reference to the executor where we + // should fetch the data + if rt.LogsFetchFinished() && rt.ArchivesFetchFinished() { + if err := store.DeleteExecutorTask(ctx, s.e, rt.ID); err != nil { + return err + } + } } } } @@ -1299,17 +1308,6 @@ func (s *Runservice) finishedRunArchiver(ctx context.Context, r *types.Run) erro } log.Infof("run %q archiving completed", r.ID) - // if the fetching is finished we can remove the executor tasks. We cannot - // remove it before since it contains the reference to the executor where we - // should fetch the data - - for _, rt := range r.Tasks { - log.Infof("deleting executor task %s", rt.ID) - if err := store.DeleteExecutorTask(ctx, s.e, rt.ID); err != nil { - return err - } - } - r.Archived = true if _, err := store.AtomicPutRun(ctx, s.e, r, nil, nil); err != nil { return err