Implement archive cleanup (#885 An error occurred )

* Implement archive cleanup

Fixes #769
  
An error occurred
 


Signed-off-by: Andrew <write@imaginarycode.com>

* Make sure to close the directory file

* Resolve issues noted by @strk

* edit cheatsheet app.ini [ci skip]

* oops [ci skip]

conf/app.ini

@@ -391,6 +391,13 @@ ARGS =
 RUN_AT_START = true
 SCHEDULE = @every 24h
 
+; Clean up old repository archives
+[cron.archive_cleanup]
+RUN_AT_START = true
+SCHEDULE = @every 24h
+; Archives created more than OLDER_THAN ago are subject to deletion
+OLDER_THAN = 24h
+
 [git]
 ; Disables highlight of added and removed changes
 DISABLE_DIFF_HIGHLIGHT = false

models/repo.go

@@ -1837,6 +1837,60 @@ func DeleteRepositoryArchives() error {
 			})
 }
 
+// DeleteOldRepositoryArchives deletes old repository archives.
+func DeleteOldRepositoryArchives() {
+	if taskStatusTable.IsRunning(archiveCleanup) {
+		return
+	}
+	taskStatusTable.Start(archiveCleanup)
+	defer taskStatusTable.Stop(archiveCleanup)
+
+	log.Trace("Doing: ArchiveCleanup")
+
+	if err := x.Where("id > 0").Iterate(new(Repository), deleteOldRepositoryArchives); err != nil {
+		log.Error(4, "ArchiveClean: %v", err)
+	}
+}
+
+func deleteOldRepositoryArchives(idx int, bean interface{}) error {
+	repo := bean.(*Repository)
+	basePath := filepath.Join(repo.RepoPath(), "archives")
+
+	for _, ty := range []string{"zip", "targz"} {
+		path := filepath.Join(basePath, ty)
+		file, err := os.Open(path)
+		if err != nil {
+			if !os.IsNotExist(err) {
+				log.Warn("Unable to open directory %s: %v", path, err)
+				return err
+			}
+
+			// If the directory doesn't exist, that's okay.
+			continue
+		}
+
+		files, err := file.Readdir(0)
+		file.Close()
+		if err != nil {
+			log.Warn("Unable to read directory %s: %v", path, err)
+			return err
+		}
+
+		minimumOldestTime := time.Now().Add(-setting.Cron.ArchiveCleanup.OlderThan)
+		for _, info := range files {
+			if info.ModTime().Before(minimumOldestTime) && !info.IsDir() {
+				toDelete := filepath.Join(path, info.Name())
+				// This is a best-effort purge, so we do not check error codes to confirm removal.
+				if err = os.Remove(toDelete); err != nil {
+					log.Trace("Unable to delete %s, but proceeding: %v", toDelete, err)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
 func gatherMissingRepoRecords() ([]*Repository, error) {
 	repos := make([]*Repository, 0, 10)
 	if err := x.
@@ -1915,9 +1969,10 @@ func RewriteRepositoryUpdateHook() error {
 var taskStatusTable = sync.NewStatusTable()
 
 const (
-	mirrorUpdate = "mirror_update"
+	mirrorUpdate   = "mirror_update"
-	gitFsck      = "git_fsck"
+	gitFsck        = "git_fsck"
-	checkRepos   = "check_repos"
+	checkRepos     = "check_repos"
+	archiveCleanup = "archive_cleanup"
 )
 
 // GitFsck calls 'git fsck' to check repository health.

modules/cron/cron.go

@@ -55,6 +55,17 @@ func NewContext() {
 			go models.CheckRepoStats()
 		}
 	}
+	if setting.Cron.ArchiveCleanup.Enabled {
+		entry, err = c.AddFunc("Clean up old repository archives", setting.Cron.ArchiveCleanup.Schedule, models.DeleteOldRepositoryArchives)
+		if err != nil {
+			log.Fatal(4, "Cron[Clean up old repository archives]: %v", err)
+		}
+		if setting.Cron.ArchiveCleanup.RunAtStart {
+			entry.Prev = time.Now()
+			entry.ExecTimes++
+			go models.DeleteOldRepositoryArchives()
+		}
+	}
 	c.Start()
 }
 

modules/setting/setting.go

@@ -307,6 +307,12 @@ var (
 			RunAtStart bool
 			Schedule   string
 		} `ini:"cron.check_repo_stats"`
+		ArchiveCleanup struct {
+			Enabled    bool
+			RunAtStart bool
+			Schedule   string
+			OlderThan  time.Duration
+		} `ini:"cron.archive_cleanup"`
 	}{
 		UpdateMirror: struct {
 			Enabled    bool
@@ -334,6 +340,16 @@ var (
 			RunAtStart: true,
 			Schedule:   "@every 24h",
 		},
+		ArchiveCleanup: struct {
+			Enabled    bool
+			RunAtStart bool
+			Schedule   string
+			OlderThan  time.Duration
+		}{
+			RunAtStart: true,
+			Schedule:   "@every 24h",
+			OlderThan:  24 * time.Hour,
+		},
 	}
 
 	// Git settings