Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions cmd/bisync/bisync_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,9 @@ var logHoppers = []string{

// Directory modification time setting can happen in any order
`INFO : .*: (Set directory modification time|Made directory with metadata).*`,

// Tracked renames can finish in any order.
`INFO : .*: Renamed from .*`,
}

// Some log lines can contain Windows path separator that must be
Expand Down Expand Up @@ -1002,6 +1005,12 @@ func (b *bisyncTest) checkPreReqs(ctx context.Context, opt *bisync.Options) (con
if strings.HasPrefix(b.fs2.String(), "sftp") {
b.fs2.Features().Disable("Copy") // disable --sftp-copy-is-hardlink as hardlinks are not truly copies
}
if b.testCase == "max_delete_track_renames" && (!operations.CanServerSideMove(b.fs1) || !operations.CanServerSideMove(b.fs2)) {
b.t.Skip("skipping test as at least one remote does not support server-side move or copy")
}
Comment thread
nielash marked this conversation as resolved.
if b.testCase == "max_delete_track_renames" && b.fs1.Hashes().Overlap(b.fs2.Hashes()).GetOne() == hash.None {
b.t.Skip("skipping test as the two remotes have no hash in common")
}
if strings.Contains(strings.ToLower(fs.ConfigString(b.fs1)), "mailru") || strings.Contains(strings.ToLower(fs.ConfigString(b.fs2)), "mailru") {
fs.GetConfig(ctx).TPSLimit = 10 // https://github.com/rclone/rclone/issues/7768#issuecomment-2060888980
}
Expand Down Expand Up @@ -1152,6 +1161,12 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
case "max-delete":
opt.MaxDelete, err = strconv.Atoi(val)
require.NoError(b.t, err, "parsing max-delete=%q", val)
case "max-delete-renames-aware":
opt.MaxDeleteRenamesAware = true
case "track-renames":
ci.TrackRenames = true
case "track-renames-strategy":
ci.TrackRenamesStrategy = val
case "size-only":
ci.SizeOnly = true
case "ignore-size":
Expand Down Expand Up @@ -1647,6 +1662,10 @@ func (b *bisyncTest) mangleResult(dir, file string, golden bool) string {
)
}
rep := logReplacements
if b.testCase == "max_delete_track_renames" && (b.fs1.Features().Move == nil || b.fs2.Features().Move == nil) {
// Without server-side Move, a tracked rename is a server-side copy + delete, counted as a transfer.
rep = append(rep, `^.*There was nothing to transfer.*$`, dropMe)
}
if b.testCase == "dry_run" {
rep = append(rep, dryrunReplacements...)
}
Expand Down
2 changes: 2 additions & 0 deletions cmd/bisync/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ type Options struct {
CreateEmptySrcDirs bool
RemoveEmptyDirs bool
MaxDelete int // percentage from 0 to 100
MaxDeleteRenamesAware bool
Force bool
FiltersFile string
Workdir string
Expand Down Expand Up @@ -130,6 +131,7 @@ func init() {
flags.BoolVarP(cmdFlags, &Opt.CheckAccess, "check-access", "", Opt.CheckAccess, MakeHelp("Ensure expected {CHECKFILE} files are found on both Path1 and Path2 filesystems, else abort."), "")
flags.StringVarP(cmdFlags, &Opt.CheckFilename, "check-filename", "", Opt.CheckFilename, MakeHelp("Filename for --check-access (default: {CHECKFILE})"), "")
flags.BoolVarP(cmdFlags, &Opt.Force, "force", "", Opt.Force, "Bypass --max-delete safety check and run the sync. Consider using with --verbose", "")
flags.BoolVarP(cmdFlags, &Opt.MaxDeleteRenamesAware, "max-delete-renames-aware", "", Opt.MaxDeleteRenamesAware, "Exclude tracked renames from the --max-delete safety check (requires --track-renames)", "")
flags.FVarP(cmdFlags, &Opt.CheckSync, "check-sync", "", "Controls comparison of final listings: true|false|only (default: true)", "")
flags.BoolVarP(cmdFlags, &Opt.CreateEmptySrcDirs, "create-empty-src-dirs", "", Opt.CreateEmptySrcDirs, "Sync creation and deletion of empty directories. (Not compatible with --remove-empty-dirs)", "")
flags.BoolVarP(cmdFlags, &Opt.RemoveEmptyDirs, "remove-empty-dirs", "", Opt.RemoveEmptyDirs, "Remove ALL empty directories at the final cleanup step.", "")
Expand Down
14 changes: 12 additions & 2 deletions cmd/bisync/compare.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ type CompareOpt = struct {

func (b *bisyncRun) setCompareDefaults(ctx context.Context) (err error) {
ci := fs.GetConfig(ctx)
renameHash := b.trackRenamesPreflight() && b.trackRenamesStrategy.UsesHash()

// defaults
b.opt.Compare.Size = true
Expand Down Expand Up @@ -58,7 +59,16 @@ func (b *bisyncRun) setCompareDefaults(ctx context.Context) (err error) {
if b.fs1.Features().SlowHash || b.fs2.Features().SlowHash {
b.opt.Compare.SlowHashDetected = true
}
if b.opt.Compare.Checksum && !b.opt.IgnoreListingChecksum {
if renameHash && b.opt.IgnoreListingChecksum {
return errors.New("--max-delete-renames-aware with a hash strategy is incompatible with --ignore-listing-checksum")
}
if renameHash && b.opt.Compare.SlowHashDetected && (b.opt.Compare.NoSlowHash || b.opt.Compare.SlowHashSyncOnly) {
return errors.New("--max-delete-renames-aware with a hash strategy is incompatible with the configured slow-hash exclusions")
}
if renameHash && b.fs1.Hashes().Overlap(b.fs2.Hashes()).GetOne() == hash.None {
return errors.New("--max-delete-renames-aware with a hash strategy requires a common hash on both paths")
}
if (b.opt.Compare.Checksum || renameHash) && !b.opt.IgnoreListingChecksum {
b.setHashType(ci)
}

Expand Down Expand Up @@ -86,7 +96,7 @@ func (b *bisyncRun) setCompareDefaults(ctx context.Context) (err error) {
// note: --checksum will still affect the internal sync calls
}
}
if !ci.CheckSum && !b.opt.Compare.Checksum && !b.opt.IgnoreListingChecksum {
if !ci.CheckSum && !b.opt.Compare.Checksum && !b.opt.IgnoreListingChecksum && !renameHash {
fs.Infoc(nil, Color(terminal.Dim, "Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set."))
b.opt.IgnoreListingChecksum = true
}
Expand Down
51 changes: 34 additions & 17 deletions cmd/bisync/deltas.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,17 +44,18 @@ func (d delta) is(cond delta) bool {

// deltaSet
type deltaSet struct {
deltas map[string]delta
size map[string]int64
time map[string]time.Time
hash map[string]string
opt *Options
fs fs.Fs // base filesystem
msg string // filesystem name for logging
oldCount int // original number of files (for "excess deletes" check)
deleted int // number of deleted files (for "excess deletes" check)
foundSame bool // true if found at least one unchanged file
checkFiles bilib.Names
deltas map[string]delta
size map[string]int64
time map[string]time.Time
hash map[string]string
opt *Options
fs fs.Fs // base filesystem
msg string // filesystem name for logging
oldCount int // original number of files (for "excess deletes" check)
deleted int // number of deleted files (for "excess deletes" check)
renameExemptions int // tracked renames excluded from the max-delete check
foundSame bool // true if found at least one unchanged file
checkFiles bilib.Names
}

func (ds *deltaSet) empty() bool {
Expand Down Expand Up @@ -541,22 +542,38 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
return
}

// excessDeletes checks whether number of deletes is within allowed range
func (ds *deltaSet) excessDeletes() bool {
func (ds *deltaSet) effectiveDeletes() int {
return max(0, ds.deleted-ds.renameExemptions)
}

func (ds *deltaSet) exceedsDeletes() bool {
maxDelete := ds.opt.MaxDelete
maxRatio := float64(maxDelete) / 100.0
curRatio := 0.0
if ds.deleted > 0 && ds.oldCount > 0 {
curRatio = float64(ds.deleted) / float64(ds.oldCount)
effectiveDeletes := ds.effectiveDeletes()
if effectiveDeletes > 0 && ds.oldCount > 0 {
curRatio = float64(effectiveDeletes) / float64(ds.oldCount)
}
return curRatio > maxRatio
}

if curRatio <= maxRatio {
// excessDeletes checks whether number of deletes is within allowed range
func (ds *deltaSet) excessDeletes() bool {
if !ds.exceedsDeletes() {
return false
}

maxDelete := ds.opt.MaxDelete
effectiveDeletes := ds.effectiveDeletes()
if ds.renameExemptions > 0 {
fs.Errorf("Safety abort",
"too many deletes (>%d%%, %d of %d after excluding %d tracked renames) on %s %s. Run with --force if desired.",
maxDelete, effectiveDeletes, ds.oldCount, ds.renameExemptions, ds.msg, quotePath(bilib.FsPath(ds.fs)))
return true
}
fs.Errorf("Safety abort",
"too many deletes (>%d%%, %d of %d) on %s %s. Run with --force if desired.",
maxDelete, ds.deleted, ds.oldCount, ds.msg, quotePath(bilib.FsPath(ds.fs)))
maxDelete, effectiveDeletes, ds.oldCount, ds.msg, quotePath(bilib.FsPath(ds.fs)))
return true
}

Expand Down
2 changes: 1 addition & 1 deletion cmd/bisync/march.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ func (b *bisyncRun) ForObject(o fs.Object, isPath1 bool) {
b.march.marchErrLock.Unlock()

var modtime time.Time
if b.opt.Compare.Modtime {
if b.opt.Compare.Modtime || (b.trackRenamesPreflight() && b.trackRenamesStrategy.UsesModtime()) {
modtime = o.ModTime(b.march.marchCtx).In(TZ)
}
id := "" // TODO: ID(o)
Expand Down
79 changes: 51 additions & 28 deletions cmd/bisync/operations.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/sync/trackrenames"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/terminal"
)
Expand All @@ -28,34 +29,35 @@ var ErrBisyncAborted = errors.New("bisync aborted")

// bisyncRun keeps bisync runtime state
type bisyncRun struct {
fs1 fs.Fs
fs2 fs.Fs
abort bool
critical bool
retryable bool
basePath string
workDir string
listing1 string
listing2 string
newListing1 string
newListing2 string
aliases bilib.AliasMap
opt *Options
octx context.Context
fctx context.Context
InGracefulShutdown bool
CleanupCompleted bool
SyncCI *fs.ConfigInfo
CancelSync context.CancelFunc
DebugName string
lockFile string
renames renames
resyncIs1to2 bool
march bisyncMarch
check bisyncCheck
queueOpt bisyncQueueOpt
downloadHashOpt downloadHashOpt
lockFileOpt lockFileOpt
fs1 fs.Fs
fs2 fs.Fs
abort bool
critical bool
retryable bool
basePath string
workDir string
listing1 string
listing2 string
newListing1 string
newListing2 string
aliases bilib.AliasMap
opt *Options
octx context.Context
fctx context.Context
InGracefulShutdown bool
CleanupCompleted bool
SyncCI *fs.ConfigInfo
CancelSync context.CancelFunc
DebugName string
lockFile string
renames renames
resyncIs1to2 bool
march bisyncMarch
check bisyncCheck
queueOpt bisyncQueueOpt
downloadHashOpt downloadHashOpt
lockFileOpt lockFileOpt
trackRenamesStrategy trackrenames.Strategy
}

type queues struct {
Expand Down Expand Up @@ -85,6 +87,15 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
}
ci := fs.GetConfig(ctx)
opt.OrigBackupDir = ci.BackupDir
if opt.MaxDeleteRenamesAware {
if !ci.TrackRenames {
return errors.New("--max-delete-renames-aware requires --track-renames")
}
b.trackRenamesStrategy, err = trackrenames.ParseStrategy(ci.TrackRenamesStrategy)
if err != nil {
return err
}
}

if ci.TerminalColorMode == fs.TerminalColorModeAlways || (ci.TerminalColorMode == fs.TerminalColorModeAuto && !log.Redirected()) {
ColorsLock.Lock()
Expand Down Expand Up @@ -346,6 +357,18 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {
// Check for too many deleted files - possible error condition.
// Don't want to start deleting on the other side!
if !opt.Force {
if b.trackRenamesPreflight() && (ds1.exceedsDeletes() || ds2.exceedsDeletes()) {
ds1.renameExemptions, ds2.renameExemptions, err = b.trackedRenameExemptions(fctx, ds1, ds2)
if err != nil {
return err
}
if ds1.renameExemptions > 0 {
fs.Infof(nil, "Path1: excluding %d tracked renames from --max-delete", ds1.renameExemptions)
}
if ds2.renameExemptions > 0 {
fs.Infof(nil, "Path2: excluding %d tracked renames from --max-delete", ds2.renameExemptions)
}
}
if ds1.excessDeletes() || ds2.excessDeletes() {
b.abort = true
return errors.New("too many deletes")
Expand Down
3 changes: 3 additions & 0 deletions cmd/bisync/rc.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,9 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
if opt.Force, err = in.GetBool("force"); rc.NotErrParamNotFound(err) {
fs.Debugf("force", "optional parameter is missing. using default value: %v", opt.Force)
}
if opt.MaxDeleteRenamesAware, err = in.GetBool("maxDeleteRenamesAware"); rc.NotErrParamNotFound(err) {
fs.Debugf("maxDeleteRenamesAware", "optional parameter is missing. using default value: %v", opt.MaxDeleteRenamesAware)
}
if opt.CreateEmptySrcDirs, err = in.GetBool("createEmptySrcDirs"); rc.NotErrParamNotFound(err) {
fs.Debugf("createEmptySrcDirs", "optional parameter is missing. using default value: %v", opt.CreateEmptySrcDirs)
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
"file6.txt"
"file7.txt"
"moved-file1.txt"
"renamed2-file1.txt"
"renamed2-file2.txt"
"renamed2-file3.txt"
"renamed2-file4.txt"
"renamed2-file5.txt"
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
"renamed-file1.txt"
"renamed-file2.txt"
"renamed-file3.txt"
"renamed-file4.txt"
"renamed-file5.txt"
"renamed2-file1.txt"
"renamed2-file2.txt"
"renamed2-file3.txt"
"renamed2-file4.txt"
"renamed2-file5.txt"
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
"renamed-file1.txt"
"renamed-file2.txt"
"renamed-file3.txt"
"renamed-file4.txt"
"renamed-file5.txt"
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
"file6.txt"
"file7.txt"
"renamed2-file1.txt"
"renamed2-file2.txt"
"renamed2-file3.txt"
"renamed2-file4.txt"
"renamed2-file5.txt"
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# bisync listing v1 from test
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file9.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "moved-file1.txt"
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# bisync listing v1 from test
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file9.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "moved-file1.txt"
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# bisync listing v1 from test
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file9.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "renamed2-file1.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "renamed2-file2.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "renamed2-file3.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "renamed2-file4.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "renamed2-file5.txt"
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# bisync listing v1 from test
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file9.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "moved-file1.txt"
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# bisync listing v1 from test
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file9.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "renamed2-file1.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "renamed2-file2.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "renamed2-file3.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "renamed2-file4.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "renamed2-file5.txt"
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# bisync listing v1 from test
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file9.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "renamed2-file1.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "renamed2-file2.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "renamed2-file3.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "renamed2-file4.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "renamed2-file5.txt"
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# bisync listing v1 from test
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt"
- 20 - - 2000-01-01T00:00:00.000000000+0000 "file9.txt"
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# bisync listing v1 from test
Loading
Loading