Merge fd5b92e5ca into 1807d269cd

changelog/unreleased/issue-5689

@@ -0,0 +1,8 @@
+Enhancement: Show progress bar for `restic stats`
+
+Previously, running `restic stats` would only give progress updates for loading the index.
+Now it displays progress for how many snapshots, files, and blobs were processed so far.
+This lets users better understand if the command is working as expected or where it is hanging.
+
+https://github.com/restic/restic/issues/5689
+https://github.com/restic/restic/pull/55555

cmd/restic/cmd_stats.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"path/filepath"
 	"strings"
+	"sync"
+	"time"
 
 	"github.com/restic/chunker"
 	"github.com/restic/restic/internal/crypto"
@@ -134,8 +136,21 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
 		SnapshotsCount: 0,
 	}
 
+	var snapshots data.Snapshots
 	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args, printer) {
-		err = statsWalkSnapshot(ctx, sn, repo, opts, stats)
+		snapshots = append(snapshots, sn)
+	}
+
+	statsProgress := newStatsProgress(term, uint64(len(snapshots)))
+
+	updater := progress.NewUpdater(ui.CalculateProgressInterval(!gopts.Quiet, gopts.JSON, term.CanUpdateStatus()), func(runtime time.Duration, final bool) {
+		statsProgress.printProgress(runtime, final)
+	})
+
+	defer updater.Done()
+
+	for _, sn := range snapshots {
+		err = statsWalkSnapshot(ctx, sn, repo, opts, stats, statsProgress)
 		if err != nil {
 			return fmt.Errorf("error walking snapshot: %v", err)
 		}
@@ -160,6 +175,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
 				}
 			}
 			stats.TotalBlobCount++
+			statsProgress.update(0, 1, uint64(pbs[0].Length))
 		}
 		if stats.TotalCompressedBlobsSize > 0 {
 			stats.CompressionRatio = float64(stats.TotalCompressedBlobsUncompressedSize) / float64(stats.TotalCompressedBlobsSize)
@@ -203,7 +219,8 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
 	return nil
 }
 
-func statsWalkSnapshot(ctx context.Context, snapshot *data.Snapshot, repo restic.Loader, opts StatsOptions, stats *statsContainer) error {
+func statsWalkSnapshot(ctx context.Context, snapshot *data.Snapshot, repo restic.Loader, opts StatsOptions, stats *statsContainer, progress *statsProgress) error {
+	progress.processSnapshot()
 	if snapshot.Tree == nil {
 		return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str())
 	}
@@ -218,7 +235,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *data.Snapshot, repo restic
 
 	hardLinkIndex := restorer.NewHardlinkIndex[struct{}]()
 	err := walker.Walk(ctx, repo, *snapshot.Tree, walker.WalkVisitor{
-		ProcessNode: statsWalkTree(repo, opts, stats, hardLinkIndex),
+		ProcessNode: statsWalkTree(repo, opts, stats, hardLinkIndex, progress),
 	})
 	if err != nil {
 		return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err)
@@ -227,7 +244,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *data.Snapshot, repo restic
 	return nil
 }
 
-func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}]) walker.WalkFunc {
+func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}], progress *statsProgress) walker.WalkFunc {
 	return func(parentTreeID restic.ID, npath string, node *data.Node, nodeErr error) error {
 		if nodeErr != nil {
 			return nodeErr
@@ -235,7 +252,7 @@ func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer,
 		if node == nil {
 			return nil
 		}
-
+		progress.update(1, 0, uint64(node.Size))
 		if opts.countMode == countModeUniqueFilesByContents || opts.countMode == countModeBlobsPerFile {
 			// only count this file if we haven't visited it before
 			fid := makeFileIDByContents(node)
@@ -255,6 +272,7 @@ func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer,
 						// ensure we have this file (by path) in our map; in this
 						// mode, a file is unique by both contents and path
 						nodePath := filepath.Join(npath, node.Name)
+						progress.update(0, 1, 0)
 						if _, ok := stats.fileBlobs[nodePath]; !ok {
 							stats.fileBlobs[nodePath] = restic.NewIDSet()
 							stats.TotalFileCount++
@@ -294,7 +312,6 @@ func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer,
 				}
 			}
 		}
-
 		return nil
 	}
 }
@@ -352,6 +369,68 @@ type statsContainer struct {
 	// independent of references to files
 	blobs restic.AssociatedBlobSet
 }
+type statsProgress struct {
+	term          ui.Terminal
+	m             sync.Mutex
+	snapshotCount uint64
+
+	processedSnapshotCount uint64
+	processedFileCount     uint64
+	processedBlobCount     uint64
+	processedSize          uint64
+}
+
+func newStatsProgress(term ui.Terminal, snapshotCount uint64) *statsProgress {
+	return &statsProgress{
+		term:          term,
+		snapshotCount: snapshotCount,
+	}
+}
+
+func (s *statsProgress) printProgress(runtime time.Duration, final bool) {
+	progressBase := s.processedSnapshotCount
+	if progressBase > 0 && !final {
+		progressBase--
+	}
+
+	status := fmt.Sprintf("[%s] %s Snapshot %v / %v", ui.FormatDuration(runtime), ui.FormatPercent(progressBase, s.snapshotCount), s.processedSnapshotCount, s.snapshotCount)
+
+	if s.processedFileCount > 0 {
+		status += fmt.Sprintf(", %v files", s.processedFileCount)
+	}
+
+	if s.processedBlobCount > 0 {
+		status += fmt.Sprintf(", %d blobs", s.processedBlobCount)
+	}
+
+	status += fmt.Sprintf(", %s", ui.FormatBytes(s.processedSize))
+
+	if final {
+		s.term.SetStatus(nil)
+		s.term.Print(status)
+	} else {
+		s.term.SetStatus([]string{status})
+	}
+}
+
+func (s *statsProgress) update(fileCount uint64, blobCount uint64, size uint64) {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	s.processedFileCount += fileCount
+	s.processedBlobCount += blobCount
+	s.processedSize += size
+}
+
+func (s *statsProgress) processSnapshot() {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	s.processedSnapshotCount++
+	s.processedFileCount = 0
+	s.processedBlobCount = 0
+	s.processedSize = 0
+}
 
 // fileID is a 256-bit hash that distinguishes unique files.
 type fileID [32]byte

cmd/restic/cmd_stats_test.go

@@ -2,8 +2,10 @@ package main
 
 import (
 	"testing"
+	"time"
 
 	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui"
 )
 
 func TestSizeHistogramNew(t *testing.T) {
@@ -60,3 +62,29 @@ func TestSizeHistogramString(t *testing.T) {
 		rtest.Equals(t, "Count: 3\nTotal Size: 11 B\nSize          Count\n-------------------\n  0 - 0 Byte  1\n  1 - 9 Byte  1\n10 - 42 Byte  1\n-------------------\n", h.String())
 	})
 }
+
+func TestStatsProgress(t *testing.T) {
+	term := &ui.MockTerminal{}
+
+	progress := newStatsProgress(term, 2)
+	progress.printProgress(0*time.Second, false)
+	rtest.Equals(t, []string{"[0:00] 0.00% Snapshot 0 / 2, 0 B"}, term.Output)
+
+	progress.processSnapshot()
+	progress.update(1, 2, 3)
+	progress.printProgress(5*time.Second, false)
+	// Output differs from the previous one because the progress is based on the number of processed snapshots,
+	// Snapshot 1/2 means processing the snapshot 1 currently
+	rtest.Equals(t, []string{"[0:05] 0.00% Snapshot 1 / 2, 1 files, 2 blobs, 3 B"}, term.Output)
+
+	progress.processSnapshot()
+	progress.printProgress(10*time.Second, false)
+	rtest.Equals(t, []string{"[0:10] 50.00% Snapshot 2 / 2, 0 B"}, term.Output)
+
+	progress.update(4, 5, 6)
+	progress.printProgress(15*time.Second, false)
+	rtest.Equals(t, []string{"[0:15] 50.00% Snapshot 2 / 2, 4 files, 5 blobs, 6 B"}, term.Output)
+
+	progress.printProgress(20*time.Second, true)
+	rtest.Equals(t, []string{"[0:20] 100.00% Snapshot 2 / 2, 4 files, 5 blobs, 6 B"}, term.Output)
+}