From 6f637bed2f12887edeb01b3c507d53030b6f4cbf Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Mon, 5 Oct 2015 02:27:24 +0200
Subject: [PATCH 01/70] LoggedIO: deduplicated code, improved checks and error
 handling in read()

Code shared by read() and iter_objects() was moved into _read().

Compared to read()'s previous state, this improved:
- fixed size check to avoid read with negative size
- exception handler for struct unpack
- checking for short read
- more precise exception messages
---
 borg/repository.py | 60 +++++++++++++++++++++++++---------------------
 1 file changed, 33 insertions(+), 27 deletions(-)

diff --git a/borg/repository.py b/borg/repository.py
index ee3074311..747cc3d0a 100644
--- a/borg/repository.py
+++ b/borg/repository.py
@@ -534,26 +534,9 @@ class LoggedIO:
         offset = MAGIC_LEN
         header = fd.read(self.header_fmt.size)
         while header:
-            try:
-                crc, size, tag = self.header_fmt.unpack(header)
-            except struct.error as err:
-                raise IntegrityError('Invalid segment entry header [offset {}]: {}'.format(offset, err))
-            if size > MAX_OBJECT_SIZE or size < self.header_fmt.size:
-                raise IntegrityError('Invalid segment entry size [offset {}]'.format(offset))
-            length = size - self.header_fmt.size
-            rest = fd.read(length)
-            if len(rest) != length:
-                raise IntegrityError('Segment entry data short read [offset {}]: expected: {}, got {} bytes'.format(
-                                     offset, length, len(rest)))
-            if crc32(rest, crc32(memoryview(header)[4:])) & 0xffffffff != crc:
-                raise IntegrityError('Segment entry checksum mismatch [offset {}]'.format(offset))
-            if tag not in (TAG_PUT, TAG_DELETE, TAG_COMMIT):
-                raise IntegrityError('Invalid segment entry tag [offset {}]'.format(offset))
-            key = None
-            if tag in (TAG_PUT, TAG_DELETE):
-                key = rest[:32]
+            size, tag, key, data = self._read(fd, self.header_fmt, header, offset, (TAG_PUT, TAG_DELETE, TAG_COMMIT))
             if include_data:
-                yield tag, key, offset, rest[32:]
+                yield tag, key, offset, data
             else:
                 yield tag, key, offset
             offset += size
@@ -586,16 +569,39 @@ class LoggedIO:
         fd = self.get_fd(segment)
         fd.seek(offset)
         header = fd.read(self.put_header_fmt.size)
-        crc, size, tag, key = self.put_header_fmt.unpack(header)
-        if size > MAX_OBJECT_SIZE:
-            raise IntegrityError('Invalid segment object size')
-        data = fd.read(size - self.put_header_fmt.size)
-        if crc32(data, crc32(memoryview(header)[4:])) & 0xffffffff != crc:
-            raise IntegrityError('Segment checksum mismatch')
-        if tag != TAG_PUT or id != key:
-            raise IntegrityError('Invalid segment entry header')
+        size, tag, key, data = self._read(fd, self.put_header_fmt, header, offset, (TAG_PUT, ))
+        if id != key:
+            raise IntegrityError('Invalid segment entry header, is not for wanted id [offset {}]'.format(offset))
         return data
 
+    def _read(self, fd, fmt, header, offset, acceptable_tags):
+        # some code shared by read() and iter_objects()
+        try:
+            hdr_tuple = fmt.unpack(header)
+        except struct.error as err:
+            raise IntegrityError('Invalid segment entry header [offset {}]: {}'.format(offset, err))
+        if fmt is self.put_header_fmt:
+            crc, size, tag, key = hdr_tuple
+        elif fmt is self.header_fmt:
+            crc, size, tag = hdr_tuple
+            key = None
+        else:
+            raise TypeError("_read called with unsupported format")
+        if size > MAX_OBJECT_SIZE or size < fmt.size:
+            raise IntegrityError('Invalid segment entry size [offset {}]'.format(offset))
+        length = size - fmt.size
+        data = fd.read(length)
+        if len(data) != length:
+            raise IntegrityError('Segment entry data short read [offset {}]: expected: {}, got {} bytes'.format(
+                                 offset, length, len(data)))
+        if crc32(data, crc32(memoryview(header)[4:])) & 0xffffffff != crc:
+            raise IntegrityError('Segment entry checksum mismatch [offset {}]'.format(offset))
+        if tag not in acceptable_tags:
+            raise IntegrityError('Invalid segment entry header, did not get acceptable tag [offset {}]'.format(offset))
+        if key is None and tag in (TAG_PUT, TAG_DELETE):
+            key, data = data[:32], data[32:]
+        return size, tag, key, data
+
     def write_put(self, id, data):
         size = len(data) + self.put_header_fmt.size
         fd = self.get_write_fd()

From c50f32426b4f993a8dcc80cd6b90799f3bce7382 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Mon, 5 Oct 2015 23:23:59 +0200
Subject: [PATCH 02/70] do not crash on empty lock.roster, fixes #232

---
 borg/locking.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/borg/locking.py b/borg/locking.py
index 8e4f1a41f..b2beac345 100644
--- a/borg/locking.py
+++ b/borg/locking.py
@@ -169,6 +169,9 @@ class LockRoster:
             if err.errno != errno.ENOENT:
                 raise
             data = {}
+        except ValueError:
+            # corrupt/empty roster file?
+            data = {}
         return data
 
     def save(self, data):

From 427ddd64a6734203bc09a92473fa85fe85f645c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Mon, 5 Oct 2015 17:50:46 -0400
Subject: [PATCH 03/70] respect XDG_CACHE_HOME

fixes attic#181
---
 borg/helpers.py           |  4 ++--
 borg/testsuite/helpers.py | 12 +++++++++++-
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/borg/helpers.py b/borg/helpers.py
index f9450c1b8..47d454bec 100644
--- a/borg/helpers.py
+++ b/borg/helpers.py
@@ -172,8 +172,8 @@ def get_keys_dir():
 
 def get_cache_dir():
     """Determine where to repository keys and cache"""
-    return os.environ.get('BORG_CACHE_DIR',
-                          os.path.join(os.path.expanduser('~'), '.cache', 'borg'))
+    xdg_cache = os.environ.get('XDG_CACHE_HOME', os.path.join(os.path.expanduser('~'), '.cache'))
+    return os.environ.get('BORG_CACHE_DIR', os.path.join(xdg_cache, 'borg'))
 
 
 def to_localtime(ts):
diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py
index 25ec48c90..4d36eb9ef 100644
--- a/borg/testsuite/helpers.py
+++ b/borg/testsuite/helpers.py
@@ -1,13 +1,14 @@
 import hashlib
 from time import mktime, strptime
 from datetime import datetime, timezone, timedelta
+import os
 
 import pytest
 import sys
 import msgpack
 
 from ..helpers import adjust_patterns, exclude_path, Location, format_timedelta, IncludePattern, ExcludePattern, make_path_safe, \
-    prune_within, prune_split, \
+    prune_within, prune_split, get_cache_dir, \
     StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams
 from . import BaseTestCase
 
@@ -381,3 +382,12 @@ class TestParseTimestamp(BaseTestCase):
     def test(self):
         self.assert_equal(parse_timestamp('2015-04-19T20:25:00.226410'), datetime(2015, 4, 19, 20, 25, 0, 226410, timezone.utc))
         self.assert_equal(parse_timestamp('2015-04-19T20:25:00'), datetime(2015, 4, 19, 20, 25, 0, 0, timezone.utc))
+
+
+def test_get_cache_dir():
+    """test that get_cache_dir respects environement"""
+    assert get_cache_dir() == os.path.join(os.path.expanduser('~'), '.cache', 'borg')
+    os.environ['XDG_CACHE_HOME'] = '/var/tmp/.cache'
+    assert get_cache_dir() == os.path.join('/var/tmp/.cache', 'borg')
+    os.environ['BORG_CACHE_DIR'] = '/var/tmp'
+    assert get_cache_dir() == '/var/tmp'

From de2a81160685099a835a7f3dfa3eb3c55cb5f19f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Mon, 5 Oct 2015 18:43:54 -0400
Subject: [PATCH 04/70] move RemoteRepository defaults to the class

the reasoning behind this is that we may need to test a
RemoteRepository setup outside of the main archiver routines, which
the current default location makes impossible

by moving the umask and remote_path remotes into the RemoteRepository
the (reasonable) defaults are available regardless of the (currently
obscure) initialisation routine, and make unit tests easier to develop
and support
---
 borg/archiver.py | 8 ++++----
 borg/remote.py   | 5 +++--
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/borg/archiver.py b/borg/archiver.py
index fbdf210a1..57e30760e 100644
--- a/borg/archiver.py
+++ b/borg/archiver.py
@@ -571,10 +571,10 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
                                    help='verbose output')
         common_parser.add_argument('--no-files-cache', dest='cache_files', action='store_false',
                                    help='do not load/update the file metadata cache used to detect unchanged files')
-        common_parser.add_argument('--umask', dest='umask', type=lambda s: int(s, 8), default=0o077, metavar='M',
-                                   help='set umask to M (local and remote, default: 0o077)')
-        common_parser.add_argument('--remote-path', dest='remote_path', default='borg', metavar='PATH',
-                                   help='set remote path to executable (default: "borg")')
+        common_parser.add_argument('--umask', dest='umask', type=lambda s: int(s, 8), default=RemoteRepository.umask, metavar='M',
+                                   help='set umask to M (local and remote, default: %(default)s)')
+        common_parser.add_argument('--remote-path', dest='remote_path', default=RemoteRepository.remote_path, metavar='PATH',
+                                   help='set remote path to executable (default: "%(default)s")')
 
         # We can't use argparse for "serve" since we don't want it to show up in "Available commands"
         if args:
diff --git a/borg/remote.py b/borg/remote.py
index 3a274b214..ce77b8245 100644
--- a/borg/remote.py
+++ b/borg/remote.py
@@ -108,8 +108,9 @@ class RepositoryServer:  # pragma: no cover
 
 class RemoteRepository:
     extra_test_args = []
-    remote_path = None
-    umask = None
+    remote_path = 'borg'
+    # default umask, overriden by --umask, defaults to read/write only for owner
+    umask = 0o077
 
     class RPCError(Exception):
         def __init__(self, name):

From 43a65933f7d3e1caa664a84a130c7bd4051bff8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Mon, 5 Oct 2015 18:51:20 -0400
Subject: [PATCH 05/70] move ssh generation code to a stub, add unit test

---
 borg/remote.py               | 34 ++++++++++++++++++++++------------
 borg/testsuite/repository.py |  4 ++++
 2 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/borg/remote.py b/borg/remote.py
index ce77b8245..19a1416a0 100644
--- a/borg/remote.py
+++ b/borg/remote.py
@@ -126,19 +126,14 @@ class RemoteRepository:
         self.responses = {}
         self.unpacker = msgpack.Unpacker(use_list=False)
         self.p = None
-        # use local umask also for the remote process
-        umask = ['--umask', '%03o' % self.umask]
+        # XXX: ideally, the testsuite would subclass Repository and
+        # override ssh_cmd() instead of this crude hack, although
+        # __testsuite__ is not a valid domain name so this is pretty
+        # safe.
         if location.host == '__testsuite__':
-            args = [sys.executable, '-m', 'borg.archiver', 'serve'] + umask + self.extra_test_args
-        else:  # pragma: no cover
-            args = ['ssh']
-            if location.port:
-                args += ['-p', str(location.port)]
-            if location.user:
-                args.append('%s@%s' % (location.user, location.host))
-            else:
-                args.append('%s' % location.host)
-            args += [self.remote_path, 'serve'] + umask
+            args = [sys.executable, '-m', 'borg.archiver', 'serve' ] + self.extra_test_args
+        else:
+            args = self.ssh_cmd()
         self.p = Popen(args, bufsize=0, stdin=PIPE, stdout=PIPE)
         self.stdin_fd = self.p.stdin.fileno()
         self.stdout_fd = self.p.stdout.fileno()
@@ -161,6 +156,21 @@ class RemoteRepository:
     def __repr__(self):
         return '<%s %s>' % (self.__class__.__name__, self.location.canonical_path())
 
+    def umask_flag(self):
+        return ['--umask', '%03o' % self.umask]
+
+    def ssh_cmd(self, location):
+        args = ['ssh']
+        if location.port:
+            args += ['-p', str(location.port)]
+        if location.user:
+            args.append('%s@%s' % (location.user, location.host))
+        else:
+            args.append('%s' % location.host)
+        # use local umask also for the remote process
+        args += [self.remote_path, 'serve'] + self.umask_flag()
+        return args
+
     def call(self, cmd, *args, **kw):
         for resp in self.call_many(cmd, [args], **kw):
             return resp
diff --git a/borg/testsuite/repository.py b/borg/testsuite/repository.py
index 74996b717..5df0a6f97 100644
--- a/borg/testsuite/repository.py
+++ b/borg/testsuite/repository.py
@@ -325,6 +325,10 @@ class RemoteRepositoryTestCase(RepositoryTestCase):
     def test_invalid_rpc(self):
         self.assert_raises(InvalidRPCMethod, lambda: self.repository.call('__init__', None))
 
+    def test_ssh_cmd(self):
+        assert self.repository.umask is not None
+        assert self.repository.ssh_cmd(Location('example.com:foo')) == ['ssh', 'example.com', 'borg', 'serve'] + self.repository.umask_flag()
+
 
 class RemoteRepositoryCheckTestCase(RepositoryCheckTestCase):
 

From a0ef4e25ddbc36e2004a8fa6f035890b8cb17e0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Mon, 5 Oct 2015 18:54:00 -0400
Subject: [PATCH 06/70] add support for arbitrary SSH commands (attic#99)

while SSH options can be specified through `~/.ssh/config`, some users
may want to use a completely different SSH command for their backups,
without overriding their $PATH variable. it may also be easier to do
ad-hoc configuration and tests that way.

plus, the POLA tells us that users expects something like this to be
supported by commands that talk to ssh. it is supported by rsync, git
and so on.
---
 borg/remote.py               | 3 ++-
 borg/testsuite/repository.py | 2 ++
 docs/usage.rst               | 2 ++
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/borg/remote.py b/borg/remote.py
index 19a1416a0..c9d8145ba 100644
--- a/borg/remote.py
+++ b/borg/remote.py
@@ -3,6 +3,7 @@ import fcntl
 import msgpack
 import os
 import select
+import shlex
 from subprocess import Popen, PIPE
 import sys
 import tempfile
@@ -160,7 +161,7 @@ class RemoteRepository:
         return ['--umask', '%03o' % self.umask]
 
     def ssh_cmd(self, location):
-        args = ['ssh']
+        args = shlex.split(os.environ.get('BORG_RSH', 'ssh'))
         if location.port:
             args += ['-p', str(location.port)]
         if location.user:
diff --git a/borg/testsuite/repository.py b/borg/testsuite/repository.py
index 5df0a6f97..5a1524ed9 100644
--- a/borg/testsuite/repository.py
+++ b/borg/testsuite/repository.py
@@ -328,6 +328,8 @@ class RemoteRepositoryTestCase(RepositoryTestCase):
     def test_ssh_cmd(self):
         assert self.repository.umask is not None
         assert self.repository.ssh_cmd(Location('example.com:foo')) == ['ssh', 'example.com', 'borg', 'serve'] + self.repository.umask_flag()
+        os.environ['BORG_RSH'] = 'ssh --foo'
+        assert self.repository.ssh_cmd(Location('example.com:foo')) == ['ssh', '--foo', 'example.com', 'borg', 'serve'] + self.repository.umask_flag()
 
 
 class RemoteRepositoryCheckTestCase(RepositoryCheckTestCase):
diff --git a/docs/usage.rst b/docs/usage.rst
index 95b95d90d..6bd292e14 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -48,6 +48,8 @@ General:
         can either leave it away or abbreviate as `::`, if a positional parameter is required.
     BORG_PASSPHRASE
         When set, use the value to answer the passphrase question for encrypted repositories.
+    BORG_RSH
+        When set, use this command instead of ``ssh``.
     TMPDIR
         where temporary files are stored (might need a lot of temporary space for some operations)
 

From 8f0de2cab75eaeeff6e0649ba18b859a694dceb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Mon, 5 Oct 2015 19:05:27 -0400
Subject: [PATCH 07/70] fix tests on travis, which seem to set BORG_CACHE_DIR

---
 borg/testsuite/helpers.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py
index 4d36eb9ef..620a77c14 100644
--- a/borg/testsuite/helpers.py
+++ b/borg/testsuite/helpers.py
@@ -386,8 +386,16 @@ class TestParseTimestamp(BaseTestCase):
 
 def test_get_cache_dir():
     """test that get_cache_dir respects environement"""
+    # reset BORG_CACHE_DIR in order to test default
+    old_env = None
+    if os.environ.get('BORG_CACHE_DIR'):
+        old_env = os.environ['BORG_CACHE_DIR']
+        del(os.environ['BORG_CACHE_DIR'])
     assert get_cache_dir() == os.path.join(os.path.expanduser('~'), '.cache', 'borg')
     os.environ['XDG_CACHE_HOME'] = '/var/tmp/.cache'
     assert get_cache_dir() == os.path.join('/var/tmp/.cache', 'borg')
     os.environ['BORG_CACHE_DIR'] = '/var/tmp'
     assert get_cache_dir() == '/var/tmp'
+    # reset old env
+    if old_env is not None:
+        os.environ['BORG_CACHE_DIR'] = old_env

From a7b70d87cdbecd53decb9d5d33f407a7347b7a7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Mon, 5 Oct 2015 19:22:33 -0400
Subject: [PATCH 08/70] complete test coverage for SSH args parsing

---
 borg/remote.py               | 2 +-
 borg/testsuite/repository.py | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/borg/remote.py b/borg/remote.py
index c9d8145ba..8001abe2c 100644
--- a/borg/remote.py
+++ b/borg/remote.py
@@ -133,7 +133,7 @@ class RemoteRepository:
         # safe.
         if location.host == '__testsuite__':
             args = [sys.executable, '-m', 'borg.archiver', 'serve' ] + self.extra_test_args
-        else:
+        else:  # pragma: no cover
             args = self.ssh_cmd()
         self.p = Popen(args, bufsize=0, stdin=PIPE, stdout=PIPE)
         self.stdin_fd = self.p.stdin.fileno()
diff --git a/borg/testsuite/repository.py b/borg/testsuite/repository.py
index 5a1524ed9..2b99b83d6 100644
--- a/borg/testsuite/repository.py
+++ b/borg/testsuite/repository.py
@@ -328,6 +328,9 @@ class RemoteRepositoryTestCase(RepositoryTestCase):
     def test_ssh_cmd(self):
         assert self.repository.umask is not None
         assert self.repository.ssh_cmd(Location('example.com:foo')) == ['ssh', 'example.com', 'borg', 'serve'] + self.repository.umask_flag()
+        assert self.repository.ssh_cmd(Location('ssh://example.com/foo')) == ['ssh', 'example.com', 'borg', 'serve'] + self.repository.umask_flag()
+        assert self.repository.ssh_cmd(Location('ssh://user@example.com/foo')) == ['ssh', 'user@example.com', 'borg', 'serve'] + self.repository.umask_flag()
+        assert self.repository.ssh_cmd(Location('ssh://user@example.com:1234/foo')) == ['ssh', '-p', '1234', 'user@example.com', 'borg', 'serve'] + self.repository.umask_flag()
         os.environ['BORG_RSH'] = 'ssh --foo'
         assert self.repository.ssh_cmd(Location('example.com:foo')) == ['ssh', '--foo', 'example.com', 'borg', 'serve'] + self.repository.umask_flag()
 

From 8ddc448f41c42bb5dae71bc9f4e9d6abe801dc2c Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Tue, 6 Oct 2015 20:35:22 +0200
Subject: [PATCH 09/70] make sure to always give segment and offset in repo
 IntegrityError exception messages

this was only handled correctly at one place, by adding the segment number afterwards.
now the segment number is always included.
---
 borg/repository.py | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/borg/repository.py b/borg/repository.py
index 747cc3d0a..932e4fef3 100644
--- a/borg/repository.py
+++ b/borg/repository.py
@@ -301,7 +301,7 @@ class Repository:
             try:
                 objects = list(self.io.iter_objects(segment))
             except IntegrityError as err:
-                report_error('Error reading segment {}: {}'.format(segment, err))
+                report_error(str(err))
                 objects = []
                 if repair:
                     self.io.recover_segment(segment, filename)
@@ -530,11 +530,12 @@ class LoggedIO:
         fd = self.get_fd(segment)
         fd.seek(0)
         if fd.read(MAGIC_LEN) != MAGIC:
-            raise IntegrityError('Invalid segment magic')
+            raise IntegrityError('Invalid segment magic [segment {}, offset {}]'.format(segment, 0))
         offset = MAGIC_LEN
         header = fd.read(self.header_fmt.size)
         while header:
-            size, tag, key, data = self._read(fd, self.header_fmt, header, offset, (TAG_PUT, TAG_DELETE, TAG_COMMIT))
+            size, tag, key, data = self._read(fd, self.header_fmt, header, segment, offset,
+                                              (TAG_PUT, TAG_DELETE, TAG_COMMIT))
             if include_data:
                 yield tag, key, offset, data
             else:
@@ -569,17 +570,19 @@ class LoggedIO:
         fd = self.get_fd(segment)
         fd.seek(offset)
         header = fd.read(self.put_header_fmt.size)
-        size, tag, key, data = self._read(fd, self.put_header_fmt, header, offset, (TAG_PUT, ))
+        size, tag, key, data = self._read(fd, self.put_header_fmt, header, segment, offset, (TAG_PUT, ))
         if id != key:
-            raise IntegrityError('Invalid segment entry header, is not for wanted id [offset {}]'.format(offset))
+            raise IntegrityError('Invalid segment entry header, is not for wanted id [segment {}, offset {}]'.format(
+                segment, offset))
         return data
 
-    def _read(self, fd, fmt, header, offset, acceptable_tags):
+    def _read(self, fd, fmt, header, segment, offset, acceptable_tags):
         # some code shared by read() and iter_objects()
         try:
             hdr_tuple = fmt.unpack(header)
         except struct.error as err:
-            raise IntegrityError('Invalid segment entry header [offset {}]: {}'.format(offset, err))
+            raise IntegrityError('Invalid segment entry header [segment {}, offset {}]: {}'.format(
+                segment, offset, err))
         if fmt is self.put_header_fmt:
             crc, size, tag, key = hdr_tuple
         elif fmt is self.header_fmt:
@@ -588,16 +591,19 @@ class LoggedIO:
         else:
             raise TypeError("_read called with unsupported format")
         if size > MAX_OBJECT_SIZE or size < fmt.size:
-            raise IntegrityError('Invalid segment entry size [offset {}]'.format(offset))
+            raise IntegrityError('Invalid segment entry size [segment {}, offset {}]'.format(
+                segment, offset))
         length = size - fmt.size
         data = fd.read(length)
         if len(data) != length:
-            raise IntegrityError('Segment entry data short read [offset {}]: expected: {}, got {} bytes'.format(
-                                 offset, length, len(data)))
+            raise IntegrityError('Segment entry data short read [segment {}, offset {}]: expected {}, got {} bytes'.format(
+                segment, offset, length, len(data)))
         if crc32(data, crc32(memoryview(header)[4:])) & 0xffffffff != crc:
-            raise IntegrityError('Segment entry checksum mismatch [offset {}]'.format(offset))
+            raise IntegrityError('Segment entry checksum mismatch [segment {}, offset {}]'.format(
+                segment, offset))
         if tag not in acceptable_tags:
-            raise IntegrityError('Invalid segment entry header, did not get acceptable tag [offset {}]'.format(offset))
+            raise IntegrityError('Invalid segment entry header, did not get acceptable tag [segment {}, offset {}]'.format(
+                segment, offset))
         if key is None and tag in (TAG_PUT, TAG_DELETE):
             key, data = data[:32], data[32:]
         return size, tag, key, data

From ee66c4c4354e7fb055485d49563be10bcd411cd8 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Tue, 6 Oct 2015 21:49:21 +0200
Subject: [PATCH 10/70] remove docs about binary wheels

we stop supporting them, because there are better alternatives:
- use a distribution package (from your linux distribution), if available
- use a pyinstaller binary provided by us (they include all you need in 1 file and
  thus have better compatibility properties and are easier to install than a wheel)
- install from source (pypi or git) if everything else fails
---
 docs/development.rst  | 14 --------------
 docs/installation.rst | 34 +---------------------------------
 2 files changed, 1 insertion(+), 47 deletions(-)

diff --git a/docs/development.rst b/docs/development.rst
index 03a4b735e..409a63bdf 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -119,23 +119,9 @@ Checklist::
   - Twitter
   - IRC channel (topic)
 
-- create binary wheels and link them from issue tracker: https://github.com/borgbackup/borg/issues/147
 - create standalone binaries and link them from issue tracker: https://github.com/borgbackup/borg/issues/214
 
 
-Creating binary wheels
-----------------------
-
-With virtual env activated::
-
-    pip install -U wheel
-    python setup.py bdist_wheel
-    ls -l dist/*.whl
-
-Note: Binary wheels are rather specific for the platform they get built on.
-      E.g. a wheel built for Ubuntu 14.04 64bit likely will not work on Centos7 64bit.
-
-
 Creating standalone binaries
 ----------------------------
 
diff --git a/docs/installation.rst b/docs/installation.rst
index e0608027c..50957d17a 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -29,11 +29,8 @@ Below, we describe different ways to install |project_name|.
   binary package (for your Linux/*BSD/OS X/... distribution).
 - **pyinstaller binary** - easy and fast, we provide a ready-to-use binary file
   that just works on the supported platforms
-- **wheel** - easy and fast, needs a platform specific borgbackup binary wheel,
-  which matches your platform [OS and CPU]).
 - **pypi** - installing a source package from pypi needs more installation steps
-  and will compile stuff - try this if there is no binary wheel that works for
-  you.
+  and will need a compiler, development headers, etc..
 - **git** - for developers and power users who want to have the latest code or
   use revision control (each release is tagged).
 
@@ -91,35 +88,6 @@ It is supposed to work without requiring installation or preparations.
 Check https://github.com/borgbackup/borg/issues/214 for available binaries.
 
 
-Debian Jessie / Ubuntu 14.04 preparations (wheel)
--------------------------------------------------
-
-.. parsed-literal::
-
-    # Python stuff we need
-    apt-get install python3 python3-pip
-
-    # Libraries we need (fuse is optional)
-    apt-get install openssl libacl1 liblz4-1 fuse
-
-
-Installation (wheel)
---------------------
-
-This uses the latest binary wheel release.
-
-.. parsed-literal::
-
-    # Check https://github.com/borgbackup/borg/issues/147 for the correct
-    # platform-specific binary wheel, download and install it:
-
-    # system-wide installation, needs sudo/root permissions:
-    sudo pip install borgbackup.whl
-
-    # home directory installation, no sudo/root needed:
-    pip install --user borgbackup.whl
-
-
 Debian Jessie / Ubuntu 14.04 preparations (git/pypi)
 ----------------------------------------------------
 

From 28a85bf0aa0359bd3ac0f1bd9d898b8e09487533 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Tue, 6 Oct 2015 21:53:20 +0200
Subject: [PATCH 11/70] update website sidebar link also

---
 docs/_themes/local/sidebarusefullinks.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/_themes/local/sidebarusefullinks.html b/docs/_themes/local/sidebarusefullinks.html
index 368dee25f..47de85364 100644
--- a/docs/_themes/local/sidebarusefullinks.html
+++ b/docs/_themes/local/sidebarusefullinks.html
@@ -5,7 +5,7 @@
 <ul>
   <li><a href="https://borgbackup.github.io/borgbackup/">Main Web Site</a></li>
   <li><a href="https://pypi.python.org/pypi/borgbackup">PyPI packages</a></li>
-  <li><a href="https://github.com/borgbackup/borg/issues/147">Binary Packages</a></li>
+  <li><a href="https://github.com/borgbackup/borg/issues/214">Binaries</a></li>
   <li><a href="https://github.com/borgbackup/borg/blob/master/CHANGES.rst">Current ChangeLog</a></li>
   <li><a href="https://github.com/borgbackup/borg">GitHub</a></li>
   <li><a href="https://github.com/borgbackup/borg/issues">Issue Tracker</a></li>

From 10db8c1d9bc5b0e4050ac4cf0a673c6ced6f27ad Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Tue, 6 Oct 2015 22:55:48 +0200
Subject: [PATCH 12/70] update CHANGES.rst

---
 CHANGES.rst | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/CHANGES.rst b/CHANGES.rst
index b2c3af457..016a55348 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,6 +1,41 @@
 Borg Changelog
 ==============
 
+Version 0.27.0
+--------------
+
+New features:
+
+- "borg upgrade" command - attic -> borg one time converter / migration, #21
+- temporary hack to avoid using lots of disk space for chunks.archive.d, #235:
+  To use it: rm -rf chunks.archive.d ; touch chunks.archive.d
+- respect XDG_CACHE_HOME, attic #181
+- add support for arbitrary SSH commands, attic #99
+- borg delete --cache-only REPO (only delete cache, not REPO), attic #123
+
+
+Bug fixes:
+
+- use Debian 7 (wheezy) to build pyinstaller borgbackup binaries, fixes slow
+  down observed when running the Centos6-built binary on Ubuntu, #222
+- do not crash on empty lock.roster, fixes #232
+- fix multiple issues with the cache config version check, #234
+- fix segment entry header size check, attic #352
+  plus other error handling improvements / code deduplication there.
+- always give segment and offset in repo IntegrityErrors
+
+
+Other changes:
+
+- stop producing binary wheels, remove docs about it, #147
+- docs:
+  - add warning about prune
+  - generate usage include files only as needed
+  - development docs: add Vagrant section
+  - update / improve / reformat FAQ
+  - hint to single-file pyinstaller binaries from README
+
+
 Version 0.26.1
 --------------
 

From 190eed6bb9b695aaab5c2d2e53a7f9cdb63e8a3d Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Tue, 6 Oct 2015 23:28:56 +0200
Subject: [PATCH 13/70] Vagrant: check out pyinstaller code from master branch

they just recently released 3.0 and that is in master now
---
 Vagrantfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index d179bdd6c..45a0e7e85 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -204,7 +204,7 @@ def install_pyinstaller(boxname)
     . borg-env/bin/activate
     git clone https://github.com/pyinstaller/pyinstaller.git
     cd pyinstaller
-    git checkout develop
+    git checkout master
     pip install -e .
   EOF
 end
@@ -216,7 +216,7 @@ def install_pyinstaller_bootloader(boxname)
     . borg-env/bin/activate
     git clone https://github.com/pyinstaller/pyinstaller.git
     cd pyinstaller
-    git checkout python3
+    git checkout master
     # build bootloader, if it is not included
     cd bootloader
     python ./waf all

From a4967ec5829ee599482d89a9b9cfc227cf16997f Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 7 Oct 2015 03:32:55 +0200
Subject: [PATCH 14/70] ssh_cmd: fix wrong caller, fixes #255

---
 borg/remote.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/borg/remote.py b/borg/remote.py
index 8001abe2c..b9847c7e4 100644
--- a/borg/remote.py
+++ b/borg/remote.py
@@ -134,7 +134,7 @@ class RemoteRepository:
         if location.host == '__testsuite__':
             args = [sys.executable, '-m', 'borg.archiver', 'serve' ] + self.extra_test_args
         else:  # pragma: no cover
-            args = self.ssh_cmd()
+            args = self.ssh_cmd(location)
         self.p = Popen(args, bufsize=0, stdin=PIPE, stdout=PIPE)
         self.stdin_fd = self.p.stdin.fileno()
         self.stdout_fd = self.p.stdout.fileno()

From 81423071d7bb8abd198d6baa98edb0361d278769 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 7 Oct 2015 03:39:46 +0200
Subject: [PATCH 15/70] vagrant: llfuse install on darwin needs pkgconfig
 installed

---
 Vagrantfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Vagrantfile b/Vagrantfile
index 45a0e7e85..72dfdfddd 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -62,6 +62,7 @@ def packages_darwin
     brew install lz4
     brew install fakeroot
     brew install git
+    brew install pkgconfig
     touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
   EOF
 end

From 6299f2d02ce2eb8a463f14c6776933d295a6f662 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 7 Oct 2015 03:42:08 +0200
Subject: [PATCH 16/70] docs: pyinstaller 3.0 is released now

this or any later 3.x or git master checkout should work.
---
 docs/development.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/development.rst b/docs/development.rst
index 409a63bdf..9b4c0d893 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -129,7 +129,7 @@ Make sure you have everything built and installed (including llfuse and fuse).
 
 With virtual env activated::
 
-  pip install pyinstaller==3.0.dev2  # or a later 3.x release or git checkout
+  pip install pyinstaller>=3.0  # or git checkout master
   pyinstaller -F -n borg-PLATFORM --hidden-import=logging.config borg/__main__.py
   ls -l dist/*
 

From 30bd38b51b682e5e0a69e24e097acbb234a42b1a Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 7 Oct 2015 15:08:09 +0200
Subject: [PATCH 17/70] update linux glibc requirement (binaries built on
 debian7 now)

---
 docs/installation.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/installation.rst b/docs/installation.rst
index 50957d17a..3b8f6cf4b 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -6,7 +6,7 @@ Installation
 
 |project_name| pyinstaller binary installation requires:
 
-* Linux: glibc >= 2.12 (ok for most supported Linux releases)
+* Linux: glibc >= 2.13 (ok for most supported Linux releases)
 * MacOS X: 10.10 (unknown whether it works for older releases)
 * FreeBSD: 10.2 (unknown whether it works for older releases)
 

From 04ac82d3e2e768f9d354013614000f7e7feb43f4 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 7 Oct 2015 15:41:17 +0200
Subject: [PATCH 18/70] do binary releases via "github releases", closes #214

---
 docs/_themes/local/sidebarusefullinks.html | 2 +-
 docs/installation.rst                      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/_themes/local/sidebarusefullinks.html b/docs/_themes/local/sidebarusefullinks.html
index 47de85364..a311fe558 100644
--- a/docs/_themes/local/sidebarusefullinks.html
+++ b/docs/_themes/local/sidebarusefullinks.html
@@ -4,8 +4,8 @@
 <h3>Useful Links</h3>
 <ul>
   <li><a href="https://borgbackup.github.io/borgbackup/">Main Web Site</a></li>
+  <li><a href="https://github.com/borgbackup/borg/releases">Releases</a></li>
   <li><a href="https://pypi.python.org/pypi/borgbackup">PyPI packages</a></li>
-  <li><a href="https://github.com/borgbackup/borg/issues/214">Binaries</a></li>
   <li><a href="https://github.com/borgbackup/borg/blob/master/CHANGES.rst">Current ChangeLog</a></li>
   <li><a href="https://github.com/borgbackup/borg">GitHub</a></li>
   <li><a href="https://github.com/borgbackup/borg/issues">Issue Tracker</a></li>
diff --git a/docs/installation.rst b/docs/installation.rst
index 3b8f6cf4b..edfb0edc0 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -85,7 +85,7 @@ For some platforms we offer a ready-to-use standalone borg binary.
 
 It is supposed to work without requiring installation or preparations.
 
-Check https://github.com/borgbackup/borg/issues/214 for available binaries.
+Check https://github.com/borgbackup/borg/releases for available binaries.
 
 
 Debian Jessie / Ubuntu 14.04 preparations (git/pypi)

From ba0aeeb33188a03e4f3b796ef4b3403506f7cb0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 09:28:41 -0400
Subject: [PATCH 19/70] some fixes to the release engineering docs

link to the locations of different tools when I know them. i marked
the ones I don't know about specially so we can document those as
well.

point to the Github releases for the standalone binaries upload
---
 docs/development.rst | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/docs/development.rst b/docs/development.rst
index 9b4c0d893..493998ea3 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -100,26 +100,27 @@ Checklist::
 - any low hanging fruit left on the issue tracker?
 - run tox on all supported platforms via vagrant, check for test fails.
 - is Travis CI happy also?
-- update CHANGES.rst (compare to git log). check version number of upcoming release.
+- update CHANGES.rst (compare to git log)
+- check version number of upcoming release
 - check MANIFEST.in and setup.py - are they complete?
 - tag the release::
 
   git tag -s -m "tagged release" 0.26.0
 
 - cd docs ; make html  # to update the usage include files
-- update website with the html
+- update website with the html (XXX: how?)
 - create a release on PyPi::
 
     python setup.py register sdist upload --identity="Thomas Waldmann" --sign
 
-- close release milestone.
+- close release milestone
 - announce on::
 
-  - mailing list
-  - Twitter
-  - IRC channel (topic)
+  - `mailing list <mailto:borgbackup@librelist.org>`_
+  - Twitter (XXX: how? where?)
+  - `IRC channel <irc://irc.freenode.net/borgbackup>`_ (change ``/topic``
 
-- create standalone binaries and link them from issue tracker: https://github.com/borgbackup/borg/issues/214
+- create standalone binaries and upload them to the Github release
 
 
 Creating standalone binaries

From 2259bc050cbfddea8b4692eec1c700ea60355aed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 09:37:59 -0400
Subject: [PATCH 20/70] more reshuffling of release docs

mention that binaries should be signed
clarify where release milestones
reword all steps to be executive
---
 docs/development.rst | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/docs/development.rst b/docs/development.rst
index 493998ea3..fda1bf36e 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -96,31 +96,35 @@ Creating a new release
 
 Checklist::
 
-- all issues for this milestone closed?
-- any low hanging fruit left on the issue tracker?
-- run tox on all supported platforms via vagrant, check for test fails.
-- is Travis CI happy also?
-- update CHANGES.rst (compare to git log)
-- check version number of upcoming release
-- check MANIFEST.in and setup.py - are they complete?
+- make sure all issues for this milestone are closed or moved them to
+  the next milestone
+- look and fix any low hanging fruit left on the issue tracker
+- run tox on all supported platforms via vagrant, check for test failures
+- check that Travis CI is also happy
+- update ``CHANGES.rst``, based on ``git log $PREVIOUS_RELEASE..``
+- check version number of upcoming release in ``CHANGES.rst``
+- verify that ``MANIFEST.in`` and ``setup.py`` are complete
 - tag the release::
 
   git tag -s -m "tagged release" 0.26.0
 
-- cd docs ; make html  # to update the usage include files
+- update usage include files::
+
+  cd docs ; make html
+
 - update website with the html (XXX: how?)
 - create a release on PyPi::
 
     python setup.py register sdist upload --identity="Thomas Waldmann" --sign
 
-- close release milestone
+- close release milestone on Github
 - announce on::
 
   - `mailing list <mailto:borgbackup@librelist.org>`_
   - Twitter (XXX: how? where?)
   - `IRC channel <irc://irc.freenode.net/borgbackup>`_ (change ``/topic``
 
-- create standalone binaries and upload them to the Github release
+- create standalone binaries (see below) and upload them to the Github release
 
 
 Creating standalone binaries
@@ -132,9 +136,10 @@ With virtual env activated::
 
   pip install pyinstaller>=3.0  # or git checkout master
   pyinstaller -F -n borg-PLATFORM --hidden-import=logging.config borg/__main__.py
-  ls -l dist/*
+  gpg --armor --detach-sign dist/borg-*
 
 If you encounter issues, see also our `Vagrantfile` for details.
 
-Note: Standalone binaries built with pyinstaller are supposed to work on same OS,
-      same architecture (x86 32bit, amd64 64bit) without external dependencies.
+.. note:: Standalone binaries built with pyinstaller are supposed to
+          work on same OS, same architecture (x86 32bit, amd64 64bit)
+          without external dependencies.

From 4dca50fafab302622eb5c98e0f26a6b34abe8ca6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 09:44:28 -0400
Subject: [PATCH 21/70] new proposal: formal release notes

this integrates the ideas in #214 to have a small checklist of things to send in the announcements on the mailing list and on the github release
---
 docs/development.rst | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/docs/development.rst b/docs/development.rst
index fda1bf36e..dff66ec54 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -113,18 +113,27 @@ Checklist::
   cd docs ; make html
 
 - update website with the html (XXX: how?)
-- create a release on PyPi::
+- write a release notes announcement, with:
+
+  1. a single summary paragraph, outlining major changes, security
+     issues, or deprecation warnings, and generally the severity of
+     the release, with a pointer to ``CHANGES.rst``
+  2. instructions for installing and upgrading borg (pointers to the
+     regular install docs, except for new installation/upgrade methods)
+  3. known issues (to be updated as we go along)
+
+- create a release on PyPi with the above release notes::
 
     python setup.py register sdist upload --identity="Thomas Waldmann" --sign
 
-- close release milestone on Github
-- announce on::
+- announce the release notes on::
 
   - `mailing list <mailto:borgbackup@librelist.org>`_
   - Twitter (XXX: how? where?)
   - `IRC channel <irc://irc.freenode.net/borgbackup>`_ (change ``/topic``
 
-- create standalone binaries (see below) and upload them to the Github release
+- create standalone binaries (see below)
+- upload standalone binaries to the Github release, integrate release notes
 
 
 Creating standalone binaries

From d375a69689851c402212e0590b069792cc9bcd4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 09:53:19 -0400
Subject: [PATCH 22/70] fix rst formatting warnings

---
 docs/development.rst | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/development.rst b/docs/development.rst
index dff66ec54..5712bfc7a 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -94,7 +94,7 @@ Usage::
 Creating a new release
 ----------------------
 
-Checklist::
+Checklist:
 
 - make sure all issues for this milestone are closed or moved them to
   the next milestone
@@ -106,11 +106,11 @@ Checklist::
 - verify that ``MANIFEST.in`` and ``setup.py`` are complete
 - tag the release::
 
-  git tag -s -m "tagged release" 0.26.0
+    git tag -s -m "tagged release" 0.26.0
 
 - update usage include files::
 
-  cd docs ; make html
+    cd docs ; make html
 
 - update website with the html (XXX: how?)
 - write a release notes announcement, with:
@@ -126,11 +126,11 @@ Checklist::
 
     python setup.py register sdist upload --identity="Thomas Waldmann" --sign
 
-- announce the release notes on::
+- announce the release notes on:
 
-  - `mailing list <mailto:borgbackup@librelist.org>`_
-  - Twitter (XXX: how? where?)
-  - `IRC channel <irc://irc.freenode.net/borgbackup>`_ (change ``/topic``
+ - `mailing list <mailto:borgbackup@librelist.org>`_
+ - Twitter (XXX: how? where?)
+ - `IRC channel <irc://irc.freenode.net/borgbackup>`_ (change ``/topic``
 
 - create standalone binaries (see below)
 - upload standalone binaries to the Github release, integrate release notes

From 74338f8a82ce5d725eca1c0769d730744bbbd3cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 09:59:30 -0400
Subject: [PATCH 23/70] update where twitter is

---
 docs/development.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/development.rst b/docs/development.rst
index 5712bfc7a..d2642d8cd 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -129,7 +129,7 @@ Checklist:
 - announce the release notes on:
 
  - `mailing list <mailto:borgbackup@librelist.org>`_
- - Twitter (XXX: how? where?)
+ - Twitter (your personnal account, if you have one)
  - `IRC channel <irc://irc.freenode.net/borgbackup>`_ (change ``/topic``
 
 - create standalone binaries (see below)

From ebe2e397f7a8a1458584257dca0845261fdb660c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 10:01:12 -0400
Subject: [PATCH 24/70] -them

---
 docs/development.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/development.rst b/docs/development.rst
index d2642d8cd..baf9d316d 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -96,8 +96,8 @@ Creating a new release
 
 Checklist:
 
-- make sure all issues for this milestone are closed or moved them to
-  the next milestone
+- make sure all issues for this milestone are closed or move to the
+  next milestone
 - look and fix any low hanging fruit left on the issue tracker
 - run tox on all supported platforms via vagrant, check for test failures
 - check that Travis CI is also happy

From a49029db1360203ba1e9f6d97b38adf069d7e9a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 10:08:00 -0400
Subject: [PATCH 25/70] s/look/find/

---
 docs/development.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/development.rst b/docs/development.rst
index baf9d316d..fc9b8b2fa 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -98,7 +98,7 @@ Checklist:
 
 - make sure all issues for this milestone are closed or move to the
   next milestone
-- look and fix any low hanging fruit left on the issue tracker
+- find and fix any low hanging fruit left on the issue tracker
 - run tox on all supported platforms via vagrant, check for test failures
 - check that Travis CI is also happy
 - update ``CHANGES.rst``, based on ``git log $PREVIOUS_RELEASE..``

From bb9b31e265381fe2cb92a661ad7450acc16a218d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 10:12:56 -0400
Subject: [PATCH 26/70] Revert "new proposal: formal release notes"

This reverts commit 4dca50fafab302622eb5c98e0f26a6b34abe8ca6.

Conflicts:
	docs/development.rst
---
 docs/development.rst | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/docs/development.rst b/docs/development.rst
index fc9b8b2fa..568f8a9d1 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -113,27 +113,18 @@ Checklist:
     cd docs ; make html
 
 - update website with the html (XXX: how?)
-- write a release notes announcement, with:
-
-  1. a single summary paragraph, outlining major changes, security
-     issues, or deprecation warnings, and generally the severity of
-     the release, with a pointer to ``CHANGES.rst``
-  2. instructions for installing and upgrading borg (pointers to the
-     regular install docs, except for new installation/upgrade methods)
-  3. known issues (to be updated as we go along)
-
-- create a release on PyPi with the above release notes::
+- create a release on PyPi::
 
     python setup.py register sdist upload --identity="Thomas Waldmann" --sign
 
-- announce the release notes on:
+- close release milestone on Github
+- announce on::
 
  - `mailing list <mailto:borgbackup@librelist.org>`_
  - Twitter (your personnal account, if you have one)
  - `IRC channel <irc://irc.freenode.net/borgbackup>`_ (change ``/topic``
 
-- create standalone binaries (see below)
-- upload standalone binaries to the Github release, integrate release notes
+- create standalone binaries (see below) and upload them to the Github release
 
 
 Creating standalone binaries

From 48c8186592c5eca272d7f864153f7ff2fab0863f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 10:16:07 -0400
Subject: [PATCH 27/70] detail what needs to happen in the github release

---
 docs/development.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/development.rst b/docs/development.rst
index 568f8a9d1..d5b77369c 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -124,7 +124,9 @@ Checklist:
  - Twitter (your personnal account, if you have one)
  - `IRC channel <irc://irc.freenode.net/borgbackup>`_ (change ``/topic``
 
-- create standalone binaries (see below) and upload them to the Github release
+- create a Github release, include:
+  * standalone binaries (see below for how to create them)
+  * a link to ``CHANGES.rst``
 
 
 Creating standalone binaries

From cec8e18d2b3b926271af0f1c95315b45679ee325 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 10:17:35 -0400
Subject: [PATCH 28/70] gpg can't sign multiple files at once, use a loop

---
 docs/development.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/development.rst b/docs/development.rst
index d5b77369c..a4efc14d4 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -138,7 +138,7 @@ With virtual env activated::
 
   pip install pyinstaller>=3.0  # or git checkout master
   pyinstaller -F -n borg-PLATFORM --hidden-import=logging.config borg/__main__.py
-  gpg --armor --detach-sign dist/borg-*
+  for file in dist/borg-*; do gpg --armor --detach-sign $file; done
 
 If you encounter issues, see also our `Vagrantfile` for details.
 

From 047e003099ddb6987f2466f33f62aaebdea72920 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Wed, 7 Oct 2015 16:55:42 +0200
Subject: [PATCH 29/70] docs: more details about release process, reordered
 sections

---
 docs/development.rst | 78 +++++++++++++++++++++++---------------------
 1 file changed, 40 insertions(+), 38 deletions(-)

diff --git a/docs/development.rst b/docs/development.rst
index a4efc14d4..b2fe34aaf 100644
--- a/docs/development.rst
+++ b/docs/development.rst
@@ -68,6 +68,11 @@ Now run::
 
 Then point a web browser at docs/_build/html/index.html.
 
+To update the web site, copy (and add, commit and push) the contents of the
+`_build` directory to the `borgbackup` directory in the web site's repository:
+https://github.com/borgbackup/borgbackup.github.io
+
+
 Using Vagrant
 -------------
 
@@ -91,48 +96,11 @@ Usage::
      vagrant scp OS:/vagrant/borg/borg/dist/borg .
 
 
-Creating a new release
-----------------------
-
-Checklist:
-
-- make sure all issues for this milestone are closed or move to the
-  next milestone
-- find and fix any low hanging fruit left on the issue tracker
-- run tox on all supported platforms via vagrant, check for test failures
-- check that Travis CI is also happy
-- update ``CHANGES.rst``, based on ``git log $PREVIOUS_RELEASE..``
-- check version number of upcoming release in ``CHANGES.rst``
-- verify that ``MANIFEST.in`` and ``setup.py`` are complete
-- tag the release::
-
-    git tag -s -m "tagged release" 0.26.0
-
-- update usage include files::
-
-    cd docs ; make html
-
-- update website with the html (XXX: how?)
-- create a release on PyPi::
-
-    python setup.py register sdist upload --identity="Thomas Waldmann" --sign
-
-- close release milestone on Github
-- announce on::
-
- - `mailing list <mailto:borgbackup@librelist.org>`_
- - Twitter (your personnal account, if you have one)
- - `IRC channel <irc://irc.freenode.net/borgbackup>`_ (change ``/topic``
-
-- create a Github release, include:
-  * standalone binaries (see below for how to create them)
-  * a link to ``CHANGES.rst``
-
-
 Creating standalone binaries
 ----------------------------
 
 Make sure you have everything built and installed (including llfuse and fuse).
+When using the Vagrant VMs, pyinstaller will already be installed.
 
 With virtual env activated::
 
@@ -145,3 +113,37 @@ If you encounter issues, see also our `Vagrantfile` for details.
 .. note:: Standalone binaries built with pyinstaller are supposed to
           work on same OS, same architecture (x86 32bit, amd64 64bit)
           without external dependencies.
+
+
+Creating a new release
+----------------------
+
+Checklist:
+
+- make sure all issues for this milestone are closed or moved to the
+  next milestone
+- find and fix any low hanging fruit left on the issue tracker
+- run tox on all supported platforms via vagrant, check for test failures
+- check that Travis CI is also happy
+- update ``CHANGES.rst``, based on ``git log $PREVIOUS_RELEASE..``
+- check version number of upcoming release in ``CHANGES.rst``
+- verify that ``MANIFEST.in`` and ``setup.py`` are complete
+- tag the release::
+
+    git tag -s -m "tagged/signed release X.Y.Z" X.Y.Z
+
+- build fresh docs and update the web site with them
+- create a release on PyPi::
+
+    python setup.py register sdist upload --identity="Thomas Waldmann" --sign
+
+- close release milestone on Github
+- announce on::
+
+ - `mailing list <mailto:borgbackup@librelist.org>`_
+ - Twitter (follow @ThomasJWaldmann for these tweets)
+ - `IRC channel <irc://irc.freenode.net/borgbackup>`_ (change ``/topic``
+
+- create a Github release, include:
+  * standalone binaries (see above for how to create them)
+  * a link to ``CHANGES.rst``

From 6629772760fe62d88cf500e857775e6aa91ed334 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 17:36:34 -0400
Subject: [PATCH 30/70] make readme a little more readable

link to source documents for people that are reading just the README without a web browser
hyperlink to the changes document as well
---
 README.rst | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 310413bfe..1fd07c2d9 100644
--- a/README.rst
+++ b/README.rst
@@ -9,7 +9,8 @@ since only changes are stored.
 The authenticated encryption technique makes it suitable for backups to not
 fully trusted targets.
 
-`Borg Installation docs <http://borgbackup.github.io/borgbackup/installation.html>`_
+See the `installation manual <installation>`_ or, if you have already
+downloaded Borg, ``docs/installation.rst`` to get started with Borg.
 
 
 Main features
@@ -115,7 +116,8 @@ considerations regarding project goals and policy of the Borg project.
 
 BORG IS NOT COMPATIBLE WITH ORIGINAL ATTIC.
 EXPECT THAT WE WILL BREAK COMPATIBILITY REPEATEDLY WHEN MAJOR RELEASE NUMBER
-CHANGES (like when going from 0.x.y to 1.0.0). Please read CHANGES document.
+CHANGES (like when going from 0.x.y to 1.0.0). Please read the
+`changelog`_ (or ``CHANGES.rst`` in the source distribution) for more information.
 
 NOT RELEASED DEVELOPMENT VERSIONS HAVE UNKNOWN COMPATIBILITY PROPERTIES.
 

From 3f2d3a8c9383f03891417ad6d605c61d3ee52cd7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 17:47:00 -0400
Subject: [PATCH 31/70] remove unneeded rst boilerplate

include CHANGES.rst directly, use a symlink to keep backwards compat
---
 CHANGES.rst      | 511 +---------------------------------------------
 docs/changes.rst | 512 ++++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 510 insertions(+), 513 deletions(-)
 mode change 100644 => 120000 CHANGES.rst

diff --git a/CHANGES.rst b/CHANGES.rst
deleted file mode 100644
index 016a55348..000000000
--- a/CHANGES.rst
+++ /dev/null
@@ -1,510 +0,0 @@
-Borg Changelog
-==============
-
-Version 0.27.0
---------------
-
-New features:
-
-- "borg upgrade" command - attic -> borg one time converter / migration, #21
-- temporary hack to avoid using lots of disk space for chunks.archive.d, #235:
-  To use it: rm -rf chunks.archive.d ; touch chunks.archive.d
-- respect XDG_CACHE_HOME, attic #181
-- add support for arbitrary SSH commands, attic #99
-- borg delete --cache-only REPO (only delete cache, not REPO), attic #123
-
-
-Bug fixes:
-
-- use Debian 7 (wheezy) to build pyinstaller borgbackup binaries, fixes slow
-  down observed when running the Centos6-built binary on Ubuntu, #222
-- do not crash on empty lock.roster, fixes #232
-- fix multiple issues with the cache config version check, #234
-- fix segment entry header size check, attic #352
-  plus other error handling improvements / code deduplication there.
-- always give segment and offset in repo IntegrityErrors
-
-
-Other changes:
-
-- stop producing binary wheels, remove docs about it, #147
-- docs:
-  - add warning about prune
-  - generate usage include files only as needed
-  - development docs: add Vagrant section
-  - update / improve / reformat FAQ
-  - hint to single-file pyinstaller binaries from README
-
-
-Version 0.26.1
---------------
-
-This is a minor update, just docs and new pyinstaller binaries.
-
-- docs update about python and binary requirements
-- better docs for --read-special, fix #220
-- re-built the binaries, fix #218 and #213 (glibc version issue)
-- update web site about single-file pyinstaller binaries
-
-Note: if you did a python-based installation, there is no need to upgrade.
-
-
-Version 0.26.0
---------------
-
-New features:
-
-- Faster cache sync (do all in one pass, remove tar/compression stuff), #163
-- BORG_REPO env var to specify the default repo, #168
-- read special files as if they were regular files, #79
-- implement borg create --dry-run, attic issue #267
-- Normalize paths before pattern matching on OS X, #143
-- support OpenBSD and NetBSD (except xattrs/ACLs)
-- support / run tests on Python 3.5
-
-Bug fixes:
-
-- borg mount repo: use absolute path, attic #200, attic #137
-- chunker: use off_t to get 64bit on 32bit platform, #178
-- initialize chunker fd to -1, so it's not equal to STDIN_FILENO (0)
-- fix reaction to "no" answer at delete repo prompt, #182
-- setup.py: detect lz4.h header file location
-- to support python < 3.2.4, add less buggy argparse lib from 3.2.6 (#194)
-- fix for obtaining 'char *' from temporary Python value (old code causes
-  a compile error on Mint 17.2)
-- llfuse 0.41 install troubles on some platforms, require < 0.41
-  (UnicodeDecodeError exception due to non-ascii llfuse setup.py)
-- cython code: add some int types to get rid of unspecific python add /
-  subtract operations (avoid undefined symbol FPE_... error on some platforms)
-- fix verbose mode display of stdin backup
-- extract: warn if a include pattern never matched, fixes #209,
-  implement counters for Include/ExcludePatterns
-- archive names with slashes are invalid, attic issue #180
-- chunker: add a check whether the POSIX_FADV_DONTNEED constant is defined -
-  fixes building on OpenBSD.
-
-Other changes:
-
-- detect inconsistency / corruption / hash collision, #170
-- replace versioneer with setuptools_scm, #106
-- docs:
-
-  - pkg-config is needed for llfuse installation
-  - be more clear about pruning, attic issue #132
-- unit tests:
-
-  - xattr: ignore security.selinux attribute showing up
-  - ext3 seems to need a bit more space for a sparse file
-  - do not test lzma level 9 compression (avoid MemoryError)
-  - work around strange mtime granularity issue on netbsd, fixes #204
-  - ignore st_rdev if file is not a block/char device, fixes #203
-  - stay away from the setgid and sticky mode bits
-- use Vagrant to do easy cross-platform testing (#196), currently:
-
-  - Debian 7 "wheezy" 32bit, Debian 8 "jessie" 64bit
-  - Ubuntu 12.04 32bit, Ubuntu 14.04 64bit
-  - Centos 7 64bit
-  - FreeBSD 10.2 64bit
-  - OpenBSD 5.7 64bit
-  - NetBSD 6.1.5 64bit
-  - Darwin (OS X Yosemite)
-
-
-Version 0.25.0
---------------
-
-Compatibility notes:
-
-- lz4 compression library (liblz4) is a new requirement (#156)
-- the new compression code is very compatible: as long as you stay with zlib
-  compression, older borg releases will still be able to read data from a
-  repo/archive made with the new code (note: this is not the case for the
-  default "none" compression, use "zlib,0" if you want a "no compression" mode
-  that can be read by older borg). Also the new code is able to read repos and
-  archives made with older borg versions (for all zlib levels  0..9).
-
-Deprecations:
-
-- --compression N (with N being a number, as in 0.24) is deprecated.
-  We keep the --compression 0..9 for now to not break scripts, but it is
-  deprecated and will be removed later, so better fix your scripts now:
-  --compression 0 (as in 0.24) is the same as --compression zlib,0 (now).
-  BUT: if you do not want compression, you rather want --compression none
-  (which is the default).
-  --compression 1 (in 0.24) is the same as --compression zlib,1 (now)
-  --compression 9 (in 0.24) is the same as --compression zlib,9 (now)
-
-New features:
-
-- create --compression none (default, means: do not compress, just pass through
-  data "as is". this is more efficient than zlib level 0 as used in borg 0.24)
-- create --compression lz4 (super-fast, but not very high compression)
-- create --compression zlib,N (slower, higher compression, default for N is 6)
-- create --compression lzma,N (slowest, highest compression, default N is 6)
-- honor the nodump flag (UF_NODUMP) and do not backup such items
-- list --short just outputs a simple list of the files/directories in an archive
-
-Bug fixes:
-
-- fixed --chunker-params parameter order confusion / malfunction, fixes #154
-- close fds of segments we delete (during compaction)
-- close files which fell out the lrucache
-- fadvise DONTNEED now is only called for the byte range actually read, not for
-  the whole file, fixes #158.
-- fix issue with negative "all archives" size, fixes #165
-- restore_xattrs: ignore if setxattr fails with EACCES, fixes #162
-
-Other changes:
-
-- remove fakeroot requirement for tests, tests run faster without fakeroot
-  (test setup does not fail any more without fakeroot, so you can run with or
-  without fakeroot), fixes #151 and #91.
-- more tests for archiver
-- recover_segment(): don't assume we have an fd for segment
-- lrucache refactoring / cleanup, add dispose function, py.test tests
-- generalize hashindex code for any key length (less hardcoding)
-- lock roster: catch file not found in remove() method and ignore it
-- travis CI: use requirements file
-- improved docs:
-
-  - replace hack for llfuse with proper solution (install libfuse-dev)
-  - update docs about compression
-  - update development docs about fakeroot
-  - internals: add some words about lock files / locking system
-  - support: mention BountySource and for what it can be used
-  - theme: use a lighter green
-  - add pypi, wheel, dist package based install docs
-  - split install docs into system-specific preparations and generic instructions
-
-
-Version 0.24.0
---------------
-
-Incompatible changes (compared to 0.23):
-
-- borg now always issues --umask NNN option when invoking another borg via ssh
-  on the repository server. By that, it's making sure it uses the same umask
-  for remote repos as for local ones. Because of this, you must upgrade both
-  server and client(s) to 0.24.
-- the default umask is 077 now (if you do not specify via --umask) which might
-  be a different one as you used previously. The default umask avoids that
-  you accidentally give access permissions for group and/or others to files
-  created by borg (e.g. the repository).
-
-Deprecations:
-
-- "--encryption passphrase" mode is deprecated, see #85 and #97.
-  See the new "--encryption repokey" mode for a replacement.
-
-New features:
-
-- borg create --chunker-params ... to configure the chunker, fixes #16
-  (attic #302, attic #300, and somehow also #41).
-  This can be used to reduce memory usage caused by chunk management overhead,
-  so borg does not create a huge chunks index/repo index and eats all your RAM
-  if you back up lots of data in huge files (like VM disk images).
-  See docs/misc/create_chunker-params.txt for more information.
-- borg info now reports chunk counts in the chunk index.
-- borg create --compression 0..9 to select zlib compression level, fixes #66
-  (attic #295).
-- borg init --encryption repokey (to store the encryption key into the repo),
-  fixes #85
-- improve at-end error logging, always log exceptions and set exit_code=1
-- LoggedIO: better error checks / exceptions / exception handling
-- implement --remote-path to allow non-default-path borg locations, #125
-- implement --umask M and use 077 as default umask for better security, #117
-- borg check: give a named single archive to it, fixes #139
-- cache sync: show progress indication
-- cache sync: reimplement the chunk index merging in C
-
-Bug fixes:
-
-- fix segfault that happened for unreadable files (chunker: n needs to be a
-  signed size_t), #116
-- fix the repair mode, #144
-- repo delete: add destroy to allowed rpc methods, fixes issue #114
-- more compatible repository locking code (based on mkdir), maybe fixes #92
-  (attic #317, attic #201).
-- better Exception msg if no Borg is installed on the remote repo server, #56
-- create a RepositoryCache implementation that can cope with >2GiB,
-  fixes attic #326.
-- fix Traceback when running check --repair, attic #232
-- clarify help text, fixes #73.
-- add help string for --no-files-cache, fixes #140
-
-Other changes:
-
-- improved docs:
-
-  - added docs/misc directory for misc. writeups that won't be included
-    "as is" into the html docs.
-  - document environment variables and return codes (attic #324, attic #52)
-  - web site: add related projects, fix web site url, IRC #borgbackup
-  - Fedora/Fedora-based install instructions added to docs
-  - Cygwin-based install instructions added to docs
-  - updated AUTHORS
-  - add FAQ entries about redundancy / integrity
-  - clarify that borg extract uses the cwd as extraction target
-  - update internals doc about chunker params, memory usage and compression
-  - added docs about development
-  - add some words about resource usage in general
-  - document how to backup a raw disk
-  - add note about how to run borg from virtual env
-  - add solutions for (ll)fuse installation problems
-  - document what borg check does, fixes #138
-  - reorganize borgbackup.github.io sidebar, prev/next at top
-  - deduplicate and refactor the docs / README.rst
-
-- use borg-tmp as prefix for temporary files / directories
-- short prune options without "keep-" are deprecated, do not suggest them
-- improved tox configuration
-- remove usage of unittest.mock, always use mock from pypi
-- use entrypoints instead of scripts, for better use of the wheel format and
-  modern installs
-- add requirements.d/development.txt and modify tox.ini
-- use travis-ci for testing based on Linux and (new) OS X
-- use coverage.py, pytest-cov and codecov.io for test coverage support
-
-I forgot to list some stuff already implemented in 0.23.0, here they are:
-
-New features:
-
-- efficient archive list from manifest, meaning a big speedup for slow
-  repo connections and "list <repo>", "delete <repo>", "prune" (attic #242,
-  attic #167)
-- big speedup for chunks cache sync (esp. for slow repo connections), fixes #18
-- hashindex: improve error messages
-
-Other changes:
-
-- explicitly specify binary mode to open binary files
-- some easy micro optimizations
-
-
-Version 0.23.0
---------------
-
-Incompatible changes (compared to attic, fork related):
-
-- changed sw name and cli command to "borg", updated docs
-- package name (and name in urls) uses "borgbackup" to have less collisions
-- changed repo / cache internal magic strings from ATTIC* to BORG*,
-  changed cache location to .cache/borg/ - this means that it currently won't
-  accept attic repos (see issue #21 about improving that)
-
-Bug fixes:
-
-- avoid defect python-msgpack releases, fixes attic #171, fixes attic #185
-- fix traceback when trying to do unsupported passphrase change, fixes attic #189
-- datetime does not like the year 10.000, fixes attic #139
-- fix "info" all archives stats, fixes attic #183
-- fix parsing with missing microseconds, fixes attic #282
-- fix misleading hint the fuse ImportError handler gave, fixes attic #237
-- check unpacked data from RPC for tuple type and correct length, fixes attic #127
-- fix Repository._active_txn state when lock upgrade fails
-- give specific path to xattr.is_enabled(), disable symlink setattr call that
-  always fails
-- fix test setup for 32bit platforms, partial fix for attic #196
-- upgraded versioneer, PEP440 compliance, fixes attic #257
-
-New features:
-
-- less memory usage: add global option --no-cache-files
-- check --last N (only check the last N archives)
-- check: sort archives in reverse time order
-- rename repo::oldname newname (rename repository)
-- create -v output more informative
-- create --progress (backup progress indicator)
-- create --timestamp (utc string or reference file/dir)
-- create: if "-" is given as path, read binary from stdin
-- extract: if --stdout is given, write all extracted binary data to stdout
-- extract --sparse (simple sparse file support)
-- extra debug information for 'fread failed'
-- delete <repo> (deletes whole repo + local cache)
-- FUSE: reflect deduplication in allocated blocks
-- only allow whitelisted RPC calls in server mode
-- normalize source/exclude paths before matching
-- use posix_fadvise to not spoil the OS cache, fixes attic #252
-- toplevel error handler: show tracebacks for better error analysis
-- sigusr1 / sigint handler to print current file infos - attic PR #286
-- RPCError: include the exception args we get from remote
-
-Other changes:
-
-- source: misc. cleanups, pep8, style
-- docs and faq improvements, fixes, updates
-- cleanup crypto.pyx, make it easier to adapt to other AES modes
-- do os.fsync like recommended in the python docs
-- source: Let chunker optionally work with os-level file descriptor.
-- source: Linux: remove duplicate os.fsencode calls
-- source: refactor _open_rb code a bit, so it is more consistent / regular
-- source: refactor indicator (status) and item processing
-- source: use py.test for better testing, flake8 for code style checks
-- source: fix tox >=2.0 compatibility (test runner)
-- pypi package: add python version classifiers, add FreeBSD to platforms
-
-
-Attic Changelog
-===============
-
-Here you can see the full list of changes between each Attic release until Borg
-forked from Attic:
-
-Version 0.17
-------------
-
-(bugfix release, released on X)
-- Fix hashindex ARM memory alignment issue (#309)
-- Improve hashindex error messages (#298)
-
-Version 0.16
-------------
-
-(bugfix release, released on May 16, 2015)
-- Fix typo preventing the security confirmation prompt from working (#303)
-- Improve handling of systems with improperly configured file system encoding (#289)
-- Fix "All archives" output for attic info. (#183)
-- More user friendly error message when repository key file is not found (#236)
-- Fix parsing of iso 8601 timestamps with zero microseconds (#282)
-
-Version 0.15
-------------
-
-(bugfix release, released on Apr 15, 2015)
-- xattr: Be less strict about unknown/unsupported platforms (#239)
-- Reduce repository listing memory usage (#163).
-- Fix BrokenPipeError for remote repositories (#233)
-- Fix incorrect behavior with two character directory names (#265, #268)
-- Require approval before accessing relocated/moved repository (#271)
-- Require approval before accessing previously unknown unencrypted repositories (#271)
-- Fix issue with hash index files larger than 2GB.
-- Fix Python 3.2 compatibility issue with noatime open() (#164)
-- Include missing pyx files in dist files (#168)
-
-Version 0.14
-------------
-
-(feature release, released on Dec 17, 2014)
-- Added support for stripping leading path segments (#95)
-  "attic extract --strip-segments X"
-- Add workaround for old Linux systems without acl_extended_file_no_follow (#96)
-- Add MacPorts' path to the default openssl search path (#101)
-- HashIndex improvements, eliminates unnecessary IO on low memory systems.
-- Fix "Number of files" output for attic info. (#124)
-- limit create file permissions so files aren't read while restoring
-- Fix issue with empty xattr values (#106)
-
-Version 0.13
-------------
-
-(feature release, released on Jun 29, 2014)
-
-- Fix sporadic "Resource temporarily unavailable" when using remote repositories
-- Reduce file cache memory usage (#90)
-- Faster AES encryption (utilizing AES-NI when available)
-- Experimental Linux, OS X and FreeBSD ACL support (#66)
-- Added support for backup and restore of BSDFlags (OSX, FreeBSD) (#56)
-- Fix bug where xattrs on symlinks were not correctly restored
-- Added cachedir support. CACHEDIR.TAG compatible cache directories
-  can now be excluded using ``--exclude-caches`` (#74)
-- Fix crash on extreme mtime timestamps (year 2400+) (#81)
-- Fix Python 3.2 specific lockf issue (EDEADLK)
-
-Version 0.12
-------------
-
-(feature release, released on April 7, 2014)
-
-- Python 3.4 support (#62)
-- Various documentation improvements a new style
-- ``attic mount`` now supports mounting an entire repository not only
-  individual archives (#59)
-- Added option to restrict remote repository access to specific path(s):
-  ``attic serve --restrict-to-path X`` (#51)
-- Include "all archives" size information in "--stats" output. (#54)
-- Added ``--stats`` option to ``attic delete`` and ``attic prune``
-- Fixed bug where ``attic prune`` used UTC instead of the local time zone
-  when determining which archives to keep.
-- Switch to SI units (Power of 1000 instead 1024) when printing file sizes
-
-Version 0.11
-------------
-
-(feature release, released on March 7, 2014)
-
-- New "check" command for repository consistency checking (#24)
-- Documentation improvements
-- Fix exception during "attic create" with repeated files (#39)
-- New "--exclude-from" option for attic create/extract/verify.
-- Improved archive metadata deduplication.
-- "attic verify" has been deprecated. Use "attic extract --dry-run" instead.
-- "attic prune --hourly|daily|..." has been deprecated.
-  Use "attic prune --keep-hourly|daily|..." instead.
-- Ignore xattr errors during "extract" if not supported by the filesystem. (#46)
-
-Version 0.10
-------------
-
-(bugfix release, released on Jan 30, 2014)
-
-- Fix deadlock when extracting 0 sized files from remote repositories
-- "--exclude" wildcard patterns are now properly applied to the full path
-  not just the file name part (#5).
-- Make source code endianness agnostic (#1)
-
-Version 0.9
------------
-
-(feature release, released on Jan 23, 2014)
-
-- Remote repository speed and reliability improvements.
-- Fix sorting of segment names to ignore NFS left over files. (#17)
-- Fix incorrect display of time (#13)
-- Improved error handling / reporting. (#12)
-- Use fcntl() instead of flock() when locking repository/cache. (#15)
-- Let ssh figure out port/user if not specified so we don't override .ssh/config (#9)
-- Improved libcrypto path detection (#23).
-
-Version 0.8.1
--------------
-
-(bugfix release, released on Oct 4, 2013)
-
-- Fix segmentation fault issue.
-
-Version 0.8
------------
-
-(feature release, released on Oct 3, 2013)
-
-- Fix xattr issue when backing up sshfs filesystems (#4)
-- Fix issue with excessive index file size (#6)
-- Support access of read only repositories.
-- New syntax to enable repository encryption:
-    attic init --encryption="none|passphrase|keyfile".
-- Detect and abort if repository is older than the cache.
-
-
-Version 0.7
------------
-
-(feature release, released on Aug 5, 2013)
-
-- Ported to FreeBSD
-- Improved documentation
-- Experimental: Archives mountable as fuse filesystems.
-- The "user." prefix is no longer stripped from xattrs on Linux
-
-
-Version 0.6.1
--------------
-
-(bugfix release, released on July 19, 2013)
-
-- Fixed an issue where mtime was not always correctly restored.
-
-
-Version 0.6
------------
-
-First public release on July 9, 2013
diff --git a/CHANGES.rst b/CHANGES.rst
new file mode 120000
index 000000000..3a83c2119
--- /dev/null
+++ b/CHANGES.rst
@@ -0,0 +1 @@
+docs/changes.rst
\ No newline at end of file
diff --git a/docs/changes.rst b/docs/changes.rst
index 5e859ecc3..016a55348 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -1,4 +1,510 @@
-.. include:: global.rst.inc
-.. _changelog:
+Borg Changelog
+==============
 
-.. include:: ../CHANGES.rst
+Version 0.27.0
+--------------
+
+New features:
+
+- "borg upgrade" command - attic -> borg one time converter / migration, #21
+- temporary hack to avoid using lots of disk space for chunks.archive.d, #235:
+  To use it: rm -rf chunks.archive.d ; touch chunks.archive.d
+- respect XDG_CACHE_HOME, attic #181
+- add support for arbitrary SSH commands, attic #99
+- borg delete --cache-only REPO (only delete cache, not REPO), attic #123
+
+
+Bug fixes:
+
+- use Debian 7 (wheezy) to build pyinstaller borgbackup binaries, fixes slow
+  down observed when running the Centos6-built binary on Ubuntu, #222
+- do not crash on empty lock.roster, fixes #232
+- fix multiple issues with the cache config version check, #234
+- fix segment entry header size check, attic #352
+  plus other error handling improvements / code deduplication there.
+- always give segment and offset in repo IntegrityErrors
+
+
+Other changes:
+
+- stop producing binary wheels, remove docs about it, #147
+- docs:
+  - add warning about prune
+  - generate usage include files only as needed
+  - development docs: add Vagrant section
+  - update / improve / reformat FAQ
+  - hint to single-file pyinstaller binaries from README
+
+
+Version 0.26.1
+--------------
+
+This is a minor update, just docs and new pyinstaller binaries.
+
+- docs update about python and binary requirements
+- better docs for --read-special, fix #220
+- re-built the binaries, fix #218 and #213 (glibc version issue)
+- update web site about single-file pyinstaller binaries
+
+Note: if you did a python-based installation, there is no need to upgrade.
+
+
+Version 0.26.0
+--------------
+
+New features:
+
+- Faster cache sync (do all in one pass, remove tar/compression stuff), #163
+- BORG_REPO env var to specify the default repo, #168
+- read special files as if they were regular files, #79
+- implement borg create --dry-run, attic issue #267
+- Normalize paths before pattern matching on OS X, #143
+- support OpenBSD and NetBSD (except xattrs/ACLs)
+- support / run tests on Python 3.5
+
+Bug fixes:
+
+- borg mount repo: use absolute path, attic #200, attic #137
+- chunker: use off_t to get 64bit on 32bit platform, #178
+- initialize chunker fd to -1, so it's not equal to STDIN_FILENO (0)
+- fix reaction to "no" answer at delete repo prompt, #182
+- setup.py: detect lz4.h header file location
+- to support python < 3.2.4, add less buggy argparse lib from 3.2.6 (#194)
+- fix for obtaining 'char *' from temporary Python value (old code causes
+  a compile error on Mint 17.2)
+- llfuse 0.41 install troubles on some platforms, require < 0.41
+  (UnicodeDecodeError exception due to non-ascii llfuse setup.py)
+- cython code: add some int types to get rid of unspecific python add /
+  subtract operations (avoid undefined symbol FPE_... error on some platforms)
+- fix verbose mode display of stdin backup
+- extract: warn if a include pattern never matched, fixes #209,
+  implement counters for Include/ExcludePatterns
+- archive names with slashes are invalid, attic issue #180
+- chunker: add a check whether the POSIX_FADV_DONTNEED constant is defined -
+  fixes building on OpenBSD.
+
+Other changes:
+
+- detect inconsistency / corruption / hash collision, #170
+- replace versioneer with setuptools_scm, #106
+- docs:
+
+  - pkg-config is needed for llfuse installation
+  - be more clear about pruning, attic issue #132
+- unit tests:
+
+  - xattr: ignore security.selinux attribute showing up
+  - ext3 seems to need a bit more space for a sparse file
+  - do not test lzma level 9 compression (avoid MemoryError)
+  - work around strange mtime granularity issue on netbsd, fixes #204
+  - ignore st_rdev if file is not a block/char device, fixes #203
+  - stay away from the setgid and sticky mode bits
+- use Vagrant to do easy cross-platform testing (#196), currently:
+
+  - Debian 7 "wheezy" 32bit, Debian 8 "jessie" 64bit
+  - Ubuntu 12.04 32bit, Ubuntu 14.04 64bit
+  - Centos 7 64bit
+  - FreeBSD 10.2 64bit
+  - OpenBSD 5.7 64bit
+  - NetBSD 6.1.5 64bit
+  - Darwin (OS X Yosemite)
+
+
+Version 0.25.0
+--------------
+
+Compatibility notes:
+
+- lz4 compression library (liblz4) is a new requirement (#156)
+- the new compression code is very compatible: as long as you stay with zlib
+  compression, older borg releases will still be able to read data from a
+  repo/archive made with the new code (note: this is not the case for the
+  default "none" compression, use "zlib,0" if you want a "no compression" mode
+  that can be read by older borg). Also the new code is able to read repos and
+  archives made with older borg versions (for all zlib levels  0..9).
+
+Deprecations:
+
+- --compression N (with N being a number, as in 0.24) is deprecated.
+  We keep the --compression 0..9 for now to not break scripts, but it is
+  deprecated and will be removed later, so better fix your scripts now:
+  --compression 0 (as in 0.24) is the same as --compression zlib,0 (now).
+  BUT: if you do not want compression, you rather want --compression none
+  (which is the default).
+  --compression 1 (in 0.24) is the same as --compression zlib,1 (now)
+  --compression 9 (in 0.24) is the same as --compression zlib,9 (now)
+
+New features:
+
+- create --compression none (default, means: do not compress, just pass through
+  data "as is". this is more efficient than zlib level 0 as used in borg 0.24)
+- create --compression lz4 (super-fast, but not very high compression)
+- create --compression zlib,N (slower, higher compression, default for N is 6)
+- create --compression lzma,N (slowest, highest compression, default N is 6)
+- honor the nodump flag (UF_NODUMP) and do not backup such items
+- list --short just outputs a simple list of the files/directories in an archive
+
+Bug fixes:
+
+- fixed --chunker-params parameter order confusion / malfunction, fixes #154
+- close fds of segments we delete (during compaction)
+- close files which fell out the lrucache
+- fadvise DONTNEED now is only called for the byte range actually read, not for
+  the whole file, fixes #158.
+- fix issue with negative "all archives" size, fixes #165
+- restore_xattrs: ignore if setxattr fails with EACCES, fixes #162
+
+Other changes:
+
+- remove fakeroot requirement for tests, tests run faster without fakeroot
+  (test setup does not fail any more without fakeroot, so you can run with or
+  without fakeroot), fixes #151 and #91.
+- more tests for archiver
+- recover_segment(): don't assume we have an fd for segment
+- lrucache refactoring / cleanup, add dispose function, py.test tests
+- generalize hashindex code for any key length (less hardcoding)
+- lock roster: catch file not found in remove() method and ignore it
+- travis CI: use requirements file
+- improved docs:
+
+  - replace hack for llfuse with proper solution (install libfuse-dev)
+  - update docs about compression
+  - update development docs about fakeroot
+  - internals: add some words about lock files / locking system
+  - support: mention BountySource and for what it can be used
+  - theme: use a lighter green
+  - add pypi, wheel, dist package based install docs
+  - split install docs into system-specific preparations and generic instructions
+
+
+Version 0.24.0
+--------------
+
+Incompatible changes (compared to 0.23):
+
+- borg now always issues --umask NNN option when invoking another borg via ssh
+  on the repository server. By that, it's making sure it uses the same umask
+  for remote repos as for local ones. Because of this, you must upgrade both
+  server and client(s) to 0.24.
+- the default umask is 077 now (if you do not specify via --umask) which might
+  be a different one as you used previously. The default umask avoids that
+  you accidentally give access permissions for group and/or others to files
+  created by borg (e.g. the repository).
+
+Deprecations:
+
+- "--encryption passphrase" mode is deprecated, see #85 and #97.
+  See the new "--encryption repokey" mode for a replacement.
+
+New features:
+
+- borg create --chunker-params ... to configure the chunker, fixes #16
+  (attic #302, attic #300, and somehow also #41).
+  This can be used to reduce memory usage caused by chunk management overhead,
+  so borg does not create a huge chunks index/repo index and eats all your RAM
+  if you back up lots of data in huge files (like VM disk images).
+  See docs/misc/create_chunker-params.txt for more information.
+- borg info now reports chunk counts in the chunk index.
+- borg create --compression 0..9 to select zlib compression level, fixes #66
+  (attic #295).
+- borg init --encryption repokey (to store the encryption key into the repo),
+  fixes #85
+- improve at-end error logging, always log exceptions and set exit_code=1
+- LoggedIO: better error checks / exceptions / exception handling
+- implement --remote-path to allow non-default-path borg locations, #125
+- implement --umask M and use 077 as default umask for better security, #117
+- borg check: give a named single archive to it, fixes #139
+- cache sync: show progress indication
+- cache sync: reimplement the chunk index merging in C
+
+Bug fixes:
+
+- fix segfault that happened for unreadable files (chunker: n needs to be a
+  signed size_t), #116
+- fix the repair mode, #144
+- repo delete: add destroy to allowed rpc methods, fixes issue #114
+- more compatible repository locking code (based on mkdir), maybe fixes #92
+  (attic #317, attic #201).
+- better Exception msg if no Borg is installed on the remote repo server, #56
+- create a RepositoryCache implementation that can cope with >2GiB,
+  fixes attic #326.
+- fix Traceback when running check --repair, attic #232
+- clarify help text, fixes #73.
+- add help string for --no-files-cache, fixes #140
+
+Other changes:
+
+- improved docs:
+
+  - added docs/misc directory for misc. writeups that won't be included
+    "as is" into the html docs.
+  - document environment variables and return codes (attic #324, attic #52)
+  - web site: add related projects, fix web site url, IRC #borgbackup
+  - Fedora/Fedora-based install instructions added to docs
+  - Cygwin-based install instructions added to docs
+  - updated AUTHORS
+  - add FAQ entries about redundancy / integrity
+  - clarify that borg extract uses the cwd as extraction target
+  - update internals doc about chunker params, memory usage and compression
+  - added docs about development
+  - add some words about resource usage in general
+  - document how to backup a raw disk
+  - add note about how to run borg from virtual env
+  - add solutions for (ll)fuse installation problems
+  - document what borg check does, fixes #138
+  - reorganize borgbackup.github.io sidebar, prev/next at top
+  - deduplicate and refactor the docs / README.rst
+
+- use borg-tmp as prefix for temporary files / directories
+- short prune options without "keep-" are deprecated, do not suggest them
+- improved tox configuration
+- remove usage of unittest.mock, always use mock from pypi
+- use entrypoints instead of scripts, for better use of the wheel format and
+  modern installs
+- add requirements.d/development.txt and modify tox.ini
+- use travis-ci for testing based on Linux and (new) OS X
+- use coverage.py, pytest-cov and codecov.io for test coverage support
+
+I forgot to list some stuff already implemented in 0.23.0, here they are:
+
+New features:
+
+- efficient archive list from manifest, meaning a big speedup for slow
+  repo connections and "list <repo>", "delete <repo>", "prune" (attic #242,
+  attic #167)
+- big speedup for chunks cache sync (esp. for slow repo connections), fixes #18
+- hashindex: improve error messages
+
+Other changes:
+
+- explicitly specify binary mode to open binary files
+- some easy micro optimizations
+
+
+Version 0.23.0
+--------------
+
+Incompatible changes (compared to attic, fork related):
+
+- changed sw name and cli command to "borg", updated docs
+- package name (and name in urls) uses "borgbackup" to have less collisions
+- changed repo / cache internal magic strings from ATTIC* to BORG*,
+  changed cache location to .cache/borg/ - this means that it currently won't
+  accept attic repos (see issue #21 about improving that)
+
+Bug fixes:
+
+- avoid defect python-msgpack releases, fixes attic #171, fixes attic #185
+- fix traceback when trying to do unsupported passphrase change, fixes attic #189
+- datetime does not like the year 10.000, fixes attic #139
+- fix "info" all archives stats, fixes attic #183
+- fix parsing with missing microseconds, fixes attic #282
+- fix misleading hint the fuse ImportError handler gave, fixes attic #237
+- check unpacked data from RPC for tuple type and correct length, fixes attic #127
+- fix Repository._active_txn state when lock upgrade fails
+- give specific path to xattr.is_enabled(), disable symlink setattr call that
+  always fails
+- fix test setup for 32bit platforms, partial fix for attic #196
+- upgraded versioneer, PEP440 compliance, fixes attic #257
+
+New features:
+
+- less memory usage: add global option --no-cache-files
+- check --last N (only check the last N archives)
+- check: sort archives in reverse time order
+- rename repo::oldname newname (rename repository)
+- create -v output more informative
+- create --progress (backup progress indicator)
+- create --timestamp (utc string or reference file/dir)
+- create: if "-" is given as path, read binary from stdin
+- extract: if --stdout is given, write all extracted binary data to stdout
+- extract --sparse (simple sparse file support)
+- extra debug information for 'fread failed'
+- delete <repo> (deletes whole repo + local cache)
+- FUSE: reflect deduplication in allocated blocks
+- only allow whitelisted RPC calls in server mode
+- normalize source/exclude paths before matching
+- use posix_fadvise to not spoil the OS cache, fixes attic #252
+- toplevel error handler: show tracebacks for better error analysis
+- sigusr1 / sigint handler to print current file infos - attic PR #286
+- RPCError: include the exception args we get from remote
+
+Other changes:
+
+- source: misc. cleanups, pep8, style
+- docs and faq improvements, fixes, updates
+- cleanup crypto.pyx, make it easier to adapt to other AES modes
+- do os.fsync like recommended in the python docs
+- source: Let chunker optionally work with os-level file descriptor.
+- source: Linux: remove duplicate os.fsencode calls
+- source: refactor _open_rb code a bit, so it is more consistent / regular
+- source: refactor indicator (status) and item processing
+- source: use py.test for better testing, flake8 for code style checks
+- source: fix tox >=2.0 compatibility (test runner)
+- pypi package: add python version classifiers, add FreeBSD to platforms
+
+
+Attic Changelog
+===============
+
+Here you can see the full list of changes between each Attic release until Borg
+forked from Attic:
+
+Version 0.17
+------------
+
+(bugfix release, released on X)
+- Fix hashindex ARM memory alignment issue (#309)
+- Improve hashindex error messages (#298)
+
+Version 0.16
+------------
+
+(bugfix release, released on May 16, 2015)
+- Fix typo preventing the security confirmation prompt from working (#303)
+- Improve handling of systems with improperly configured file system encoding (#289)
+- Fix "All archives" output for attic info. (#183)
+- More user friendly error message when repository key file is not found (#236)
+- Fix parsing of iso 8601 timestamps with zero microseconds (#282)
+
+Version 0.15
+------------
+
+(bugfix release, released on Apr 15, 2015)
+- xattr: Be less strict about unknown/unsupported platforms (#239)
+- Reduce repository listing memory usage (#163).
+- Fix BrokenPipeError for remote repositories (#233)
+- Fix incorrect behavior with two character directory names (#265, #268)
+- Require approval before accessing relocated/moved repository (#271)
+- Require approval before accessing previously unknown unencrypted repositories (#271)
+- Fix issue with hash index files larger than 2GB.
+- Fix Python 3.2 compatibility issue with noatime open() (#164)
+- Include missing pyx files in dist files (#168)
+
+Version 0.14
+------------
+
+(feature release, released on Dec 17, 2014)
+- Added support for stripping leading path segments (#95)
+  "attic extract --strip-segments X"
+- Add workaround for old Linux systems without acl_extended_file_no_follow (#96)
+- Add MacPorts' path to the default openssl search path (#101)
+- HashIndex improvements, eliminates unnecessary IO on low memory systems.
+- Fix "Number of files" output for attic info. (#124)
+- limit create file permissions so files aren't read while restoring
+- Fix issue with empty xattr values (#106)
+
+Version 0.13
+------------
+
+(feature release, released on Jun 29, 2014)
+
+- Fix sporadic "Resource temporarily unavailable" when using remote repositories
+- Reduce file cache memory usage (#90)
+- Faster AES encryption (utilizing AES-NI when available)
+- Experimental Linux, OS X and FreeBSD ACL support (#66)
+- Added support for backup and restore of BSDFlags (OSX, FreeBSD) (#56)
+- Fix bug where xattrs on symlinks were not correctly restored
+- Added cachedir support. CACHEDIR.TAG compatible cache directories
+  can now be excluded using ``--exclude-caches`` (#74)
+- Fix crash on extreme mtime timestamps (year 2400+) (#81)
+- Fix Python 3.2 specific lockf issue (EDEADLK)
+
+Version 0.12
+------------
+
+(feature release, released on April 7, 2014)
+
+- Python 3.4 support (#62)
+- Various documentation improvements a new style
+- ``attic mount`` now supports mounting an entire repository not only
+  individual archives (#59)
+- Added option to restrict remote repository access to specific path(s):
+  ``attic serve --restrict-to-path X`` (#51)
+- Include "all archives" size information in "--stats" output. (#54)
+- Added ``--stats`` option to ``attic delete`` and ``attic prune``
+- Fixed bug where ``attic prune`` used UTC instead of the local time zone
+  when determining which archives to keep.
+- Switch to SI units (Power of 1000 instead 1024) when printing file sizes
+
+Version 0.11
+------------
+
+(feature release, released on March 7, 2014)
+
+- New "check" command for repository consistency checking (#24)
+- Documentation improvements
+- Fix exception during "attic create" with repeated files (#39)
+- New "--exclude-from" option for attic create/extract/verify.
+- Improved archive metadata deduplication.
+- "attic verify" has been deprecated. Use "attic extract --dry-run" instead.
+- "attic prune --hourly|daily|..." has been deprecated.
+  Use "attic prune --keep-hourly|daily|..." instead.
+- Ignore xattr errors during "extract" if not supported by the filesystem. (#46)
+
+Version 0.10
+------------
+
+(bugfix release, released on Jan 30, 2014)
+
+- Fix deadlock when extracting 0 sized files from remote repositories
+- "--exclude" wildcard patterns are now properly applied to the full path
+  not just the file name part (#5).
+- Make source code endianness agnostic (#1)
+
+Version 0.9
+-----------
+
+(feature release, released on Jan 23, 2014)
+
+- Remote repository speed and reliability improvements.
+- Fix sorting of segment names to ignore NFS left over files. (#17)
+- Fix incorrect display of time (#13)
+- Improved error handling / reporting. (#12)
+- Use fcntl() instead of flock() when locking repository/cache. (#15)
+- Let ssh figure out port/user if not specified so we don't override .ssh/config (#9)
+- Improved libcrypto path detection (#23).
+
+Version 0.8.1
+-------------
+
+(bugfix release, released on Oct 4, 2013)
+
+- Fix segmentation fault issue.
+
+Version 0.8
+-----------
+
+(feature release, released on Oct 3, 2013)
+
+- Fix xattr issue when backing up sshfs filesystems (#4)
+- Fix issue with excessive index file size (#6)
+- Support access of read only repositories.
+- New syntax to enable repository encryption:
+    attic init --encryption="none|passphrase|keyfile".
+- Detect and abort if repository is older than the cache.
+
+
+Version 0.7
+-----------
+
+(feature release, released on Aug 5, 2013)
+
+- Ported to FreeBSD
+- Improved documentation
+- Experimental: Archives mountable as fuse filesystems.
+- The "user." prefix is no longer stripped from xattrs on Linux
+
+
+Version 0.6.1
+-------------
+
+(bugfix release, released on July 19, 2013)
+
+- Fixed an issue where mtime was not always correctly restored.
+
+
+Version 0.6
+-----------
+
+First public release on July 9, 2013

From b87be856f916b8b77e23ffae478ec79646c92eaa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 17:47:34 -0400
Subject: [PATCH 32/70] include authors, and move to docs

also keep a symlink for backwards compatibility
---
 AUTHORS          | 24 +-----------------------
 docs/authors.rst | 27 +++++++++++++++++++++++++++
 docs/index.rst   |  1 +
 3 files changed, 29 insertions(+), 23 deletions(-)
 mode change 100644 => 120000 AUTHORS
 create mode 100644 docs/authors.rst

diff --git a/AUTHORS b/AUTHORS
deleted file mode 100644
index 1f6fc9cdc..000000000
--- a/AUTHORS
+++ /dev/null
@@ -1,23 +0,0 @@
-Borg Developers / Contributors ("The Borg Collective")
-``````````````````````````````````````````````````````
-- Thomas Waldmann <tw@waldmann-edv.de>
-- Antoine Beaupré
-- Radek Podgorny <radek@podgorny.cz>
-- Yuri D'Elia
-
-Borg is a fork of Attic. Attic is written and maintained
-by Jonas Borgström and various contributors:
-
-Development Lead
-````````````````
-- Jonas Borgström <jonas@borgstrom.se>
-
-Patches and Suggestions
-```````````````````````
-- Brian Johnson
-- Cyril Roussillon
-- Dan Christensen
-- Jeremy Maitin-Shepard
-- Johann Klähn
-- Petros Moisiadis
-- Thomas Waldmann
diff --git a/AUTHORS b/AUTHORS
new file mode 120000
index 000000000..76304f50d
--- /dev/null
+++ b/AUTHORS
@@ -0,0 +1 @@
+docs/authors.rst
\ No newline at end of file
diff --git a/docs/authors.rst b/docs/authors.rst
new file mode 100644
index 000000000..353d0747f
--- /dev/null
+++ b/docs/authors.rst
@@ -0,0 +1,27 @@
+Borg Contributors ("The Borg Collective")
+=========================================
+
+- Thomas Waldmann <tw@waldmann-edv.de>
+- Antoine Beaupré
+- Radek Podgorny <radek@podgorny.cz>
+- Yuri D'Elia
+
+Attic authors
+-------------
+
+Borg is a fork of Attic. Attic is written and maintained
+by Jonas Borgström and various contributors:
+
+Development Lead
+````````````````
+- Jonas Borgström <jonas@borgstrom.se>
+
+Patches and Suggestions
+```````````````````````
+- Brian Johnson
+- Cyril Roussillon
+- Dan Christensen
+- Jeremy Maitin-Shepard
+- Johann Klähn
+- Petros Moisiadis
+- Thomas Waldmann
diff --git a/docs/index.rst b/docs/index.rst
index 6a42dce0f..cab3011fc 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -16,4 +16,5 @@ Borg Documentation
    changes
    internals
    development
+   authors
    api

From 1579be33dfebce1ae242949ea7f750dc34f683ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 17:52:40 -0400
Subject: [PATCH 33/70] fix links to changelog and authors

---
 README.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 1fd07c2d9..7108586ef 100644
--- a/README.rst
+++ b/README.rst
@@ -109,7 +109,7 @@ Notes
 -----
 
 Borg is a fork of `Attic <https://github.com/jborg/attic>`_ and maintained by
-"`The Borg Collective <https://github.com/borgbackup/borg/blob/master/AUTHORS>`_".
+":doc:`The Borg collective <authors>`".
 
 Read `issue #1 <https://github.com/borgbackup/borg/issues/1>`_ about the initial
 considerations regarding project goals and policy of the Borg project.
@@ -117,7 +117,8 @@ considerations regarding project goals and policy of the Borg project.
 BORG IS NOT COMPATIBLE WITH ORIGINAL ATTIC.
 EXPECT THAT WE WILL BREAK COMPATIBILITY REPEATEDLY WHEN MAJOR RELEASE NUMBER
 CHANGES (like when going from 0.x.y to 1.0.0). Please read the
-`changelog`_ (or ``CHANGES.rst`` in the source distribution) for more information.
+:doc:`changelog <changes>` (or ``CHANGES.rst`` in the source
+distribution) for more information.
 
 NOT RELEASED DEVELOPMENT VERSIONS HAVE UNKNOWN COMPATIBILITY PROPERTIES.
 

From 715a25802edad2f92c826e06ec454e94595b162a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 17:53:09 -0400
Subject: [PATCH 34/70] fix syntax errors in changelog

---
 docs/changes.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/changes.rst b/docs/changes.rst
index 016a55348..5f347dfea 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -70,12 +70,12 @@ Bug fixes:
 - fix reaction to "no" answer at delete repo prompt, #182
 - setup.py: detect lz4.h header file location
 - to support python < 3.2.4, add less buggy argparse lib from 3.2.6 (#194)
-- fix for obtaining 'char *' from temporary Python value (old code causes
+- fix for obtaining ``char *`` from temporary Python value (old code causes
   a compile error on Mint 17.2)
 - llfuse 0.41 install troubles on some platforms, require < 0.41
   (UnicodeDecodeError exception due to non-ascii llfuse setup.py)
 - cython code: add some int types to get rid of unspecific python add /
-  subtract operations (avoid undefined symbol FPE_... error on some platforms)
+  subtract operations (avoid ``undefined symbol FPE_``... error on some platforms)
 - fix verbose mode display of stdin backup
 - extract: warn if a include pattern never matched, fixes #209,
   implement counters for Include/ExcludePatterns
@@ -385,6 +385,7 @@ Version 0.14
 ------------
 
 (feature release, released on Dec 17, 2014)
+
 - Added support for stripping leading path segments (#95)
   "attic extract --strip-segments X"
 - Add workaround for old Linux systems without acl_extended_file_no_follow (#96)

From 60d04b05a0bff7be779f6d43a5db6a3bd934bdf9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 17:56:24 -0400
Subject: [PATCH 35/70] show the README on the homepage

instead of a boring table of contents, try to show our more exciting README file

it's still a wall of text, but at least all the buzzwords and highlights are there

ideally, the table of contents would be in the sidebar, but i don't know how to do that
---
 docs/index.rst | 3 ++-
 docs/intro.rst | 7 -------
 2 files changed, 2 insertions(+), 8 deletions(-)
 delete mode 100644 docs/intro.rst

diff --git a/docs/index.rst b/docs/index.rst
index cab3011fc..bb93e23c1 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -4,10 +4,11 @@
 Borg Documentation
 ==================
 
+.. include:: ../README.rst
+
 .. toctree::
    :maxdepth: 2
 
-   intro
    installation
    quickstart
    usage
diff --git a/docs/intro.rst b/docs/intro.rst
deleted file mode 100644
index 7e7759c7d..000000000
--- a/docs/intro.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-.. include:: global.rst.inc
-.. _foreword:
-
-Introduction
-============
-
-.. include:: ../README.rst

From 0b8e835eb8657fecad73483492c4c9615303c76a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 17:59:00 -0400
Subject: [PATCH 36/70] api.rst is autogenerated, ignore it

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 4f7c67672..9d2e6695b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,7 @@ platform_linux.c
 *.pyo
 *.so
 docs/usage/*.inc
+docs/api.rst
 .idea/
 .cache/
 borg/_version.py

From ef5d8d087910dfa6e927e8ab7c494d928873b05d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 18:04:36 -0400
Subject: [PATCH 37/70] fix link to install docs

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 7108586ef..cf7d5ca87 100644
--- a/README.rst
+++ b/README.rst
@@ -9,7 +9,7 @@ since only changes are stored.
 The authenticated encryption technique makes it suitable for backups to not
 fully trusted targets.
 
-See the `installation manual <installation>`_ or, if you have already
+See the :doc:`installation manual <installation>` or, if you have already
 downloaded Borg, ``docs/installation.rst`` to get started with Borg.
 
 

From e2f3527353cd9d1ceb6e4cd752093b197bce7cc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 18:15:30 -0400
Subject: [PATCH 38/70] don't duplicate OS list

we merge the "easy installation" and "platforms" paragraphs together in a neater phrasing
---
 README.rst | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/README.rst b/README.rst
index cf7d5ca87..4a9c7ddef 100644
--- a/README.rst
+++ b/README.rst
@@ -64,16 +64,16 @@ Main features
     Backup archives are mountable as userspace filesystems for easy interactive
     backup examination and restores (e.g. by using a regular file manager).
 
-**Easy installation**
-    For Linux, Mac OS X and FreeBSD, we offer a single-file pyinstaller binary
-    that does not require installing anything - you can just run it.
+**Easy installation on multiple platforms**
+    We offer a single-file binaries
+    that does not require installing anything - you can just run it on
+    the supported platforms:
 
-**Platforms Borg works on**
-  * Linux
-  * Mac OS X
-  * FreeBSD
-  * OpenBSD and NetBSD (for both: no xattrs/ACLs support yet)
-  * Cygwin (unsupported)
+    * Linux
+    * Mac OS X
+    * FreeBSD
+    * OpenBSD and NetBSD (for both: no xattrs/ACLs support yet)
+    * Cygwin (unsupported)
 
 **Free and Open Source Software**
   * security and functionality can be audited independently

From 60afc03d33694dcb15e8651915f3d910708e3b6d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 18:33:31 -0400
Subject: [PATCH 39/70] move authors to a stub, include LICENSE

---
 AUTHORS          | 28 +++++++++++++++++++++++++++-
 docs/authors.rst | 30 +++++++-----------------------
 2 files changed, 34 insertions(+), 24 deletions(-)
 mode change 120000 => 100644 AUTHORS

diff --git a/AUTHORS b/AUTHORS
deleted file mode 120000
index 76304f50d..000000000
--- a/AUTHORS
+++ /dev/null
@@ -1 +0,0 @@
-docs/authors.rst
\ No newline at end of file
diff --git a/AUTHORS b/AUTHORS
new file mode 100644
index 000000000..353d0747f
--- /dev/null
+++ b/AUTHORS
@@ -0,0 +1,27 @@
+Borg Contributors ("The Borg Collective")
+=========================================
+
+- Thomas Waldmann <tw@waldmann-edv.de>
+- Antoine Beaupré
+- Radek Podgorny <radek@podgorny.cz>
+- Yuri D'Elia
+
+Attic authors
+-------------
+
+Borg is a fork of Attic. Attic is written and maintained
+by Jonas Borgström and various contributors:
+
+Development Lead
+````````````````
+- Jonas Borgström <jonas@borgstrom.se>
+
+Patches and Suggestions
+```````````````````````
+- Brian Johnson
+- Cyril Roussillon
+- Dan Christensen
+- Jeremy Maitin-Shepard
+- Johann Klähn
+- Petros Moisiadis
+- Thomas Waldmann
diff --git a/docs/authors.rst b/docs/authors.rst
index 353d0747f..c368035d6 100644
--- a/docs/authors.rst
+++ b/docs/authors.rst
@@ -1,27 +1,11 @@
-Borg Contributors ("The Borg Collective")
-=========================================
+.. include:: global.rst.inc
 
-- Thomas Waldmann <tw@waldmann-edv.de>
-- Antoine Beaupré
-- Radek Podgorny <radek@podgorny.cz>
-- Yuri D'Elia
+.. include:: ../AUTHORS
 
-Attic authors
--------------
+License
+=======
 
-Borg is a fork of Attic. Attic is written and maintained
-by Jonas Borgström and various contributors:
+.. _license:
 
-Development Lead
-````````````````
-- Jonas Borgström <jonas@borgstrom.se>
-
-Patches and Suggestions
-```````````````````````
-- Brian Johnson
-- Cyril Roussillon
-- Dan Christensen
-- Jeremy Maitin-Shepard
-- Johann Klähn
-- Petros Moisiadis
-- Thomas Waldmann
+.. include:: ../LICENSE
+   :literal:

From 5f4db6487a29900a01ae661560631da7bf691e52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 18:33:43 -0400
Subject: [PATCH 40/70] fix README links so they work in github

---
 README.rst | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/README.rst b/README.rst
index 4a9c7ddef..c446ca552 100644
--- a/README.rst
+++ b/README.rst
@@ -9,9 +9,10 @@ since only changes are stored.
 The authenticated encryption technique makes it suitable for backups to not
 fully trusted targets.
 
-See the :doc:`installation manual <installation>` or, if you have already
+See the `installation manual`_ or, if you have already
 downloaded Borg, ``docs/installation.rst`` to get started with Borg.
 
+.. _installation manual: http://borgbackup.github.io/borgbackup/installation.html
 
 Main features
 ~~~~~~~~~~~~~
@@ -109,7 +110,9 @@ Notes
 -----
 
 Borg is a fork of `Attic <https://github.com/jborg/attic>`_ and maintained by
-":doc:`The Borg collective <authors>`".
+"`The Borg collective`_".
+
+.. _The Borg collective: http://borgbackup.github.io/borgbackup/authors.html
 
 Read `issue #1 <https://github.com/borgbackup/borg/issues/1>`_ about the initial
 considerations regarding project goals and policy of the Borg project.
@@ -117,15 +120,19 @@ considerations regarding project goals and policy of the Borg project.
 BORG IS NOT COMPATIBLE WITH ORIGINAL ATTIC.
 EXPECT THAT WE WILL BREAK COMPATIBILITY REPEATEDLY WHEN MAJOR RELEASE NUMBER
 CHANGES (like when going from 0.x.y to 1.0.0). Please read the
-:doc:`changelog <changes>` (or ``CHANGES.rst`` in the source
-distribution) for more information.
+`changelog`_ (or ``CHANGES.rst`` in the source distribution) for more
+information.
+
+.. _changelog: https://borgbackup.github.io/borgbackup/changes.html
 
 NOT RELEASED DEVELOPMENT VERSIONS HAVE UNKNOWN COMPATIBILITY PROPERTIES.
 
 THIS IS SOFTWARE IN DEVELOPMENT, DECIDE YOURSELF WHETHER IT FITS YOUR NEEDS.
 
-For more information, please also see the
-`LICENSE  <https://github.com/borgbackup/borg/blob/master/LICENSE>`_.
+Borg is distributed under a 3-clause BSD license, see `the license`_
+for the complete license.
+
+.. _the license: https://borgbackup.github.io/borgbackup/authors.html#license
 
 |build| |coverage|
 

From 6cd6e286af1a5b020adbb6129fdeabee870877f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 18:37:35 -0400
Subject: [PATCH 41/70] clarify which systems have binaries

---
 README.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.rst b/README.rst
index c446ca552..a43375491 100644
--- a/README.rst
+++ b/README.rst
@@ -66,15 +66,15 @@ Main features
     backup examination and restores (e.g. by using a regular file manager).
 
 **Easy installation on multiple platforms**
-    We offer a single-file binaries
+    We offer single-file binaries
     that does not require installing anything - you can just run it on
     the supported platforms:
 
     * Linux
     * Mac OS X
     * FreeBSD
-    * OpenBSD and NetBSD (for both: no xattrs/ACLs support yet)
-    * Cygwin (unsupported)
+    * OpenBSD and NetBSD (no xattrs/ACLs support or binaries yet)
+    * Cygwin (not supported, no binaries yet)
 
 **Free and Open Source Software**
   * security and functionality can be audited independently

From 28cbc6cbd1b3d70be284a69bc8049b435e101a81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 19:16:49 -0400
Subject: [PATCH 42/70] fix build on RTFD

first off, this required ticking the `Install your project inside a
virtualenv using setup.py install` box in the advanced config.

then, i had to disable all the C extensions build and disable some
checks, based on whether we are running on RTD or not.

still missing: usage builds and possibly other stuff that is in our
Makefile and not in setup.py.
---
 setup.py | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/setup.py b/setup.py
index 68a3db8d3..2e8ed1f18 100644
--- a/setup.py
+++ b/setup.py
@@ -10,6 +10,9 @@ if my_python < min_python:
     print("Borg requires Python %d.%d or later" % min_python)
     sys.exit(1)
 
+# Are we building on ReadTheDocs?
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
 # msgpack pure python data corruption was fixed in 0.4.6.
 # Also, we might use some rather recent API features.
 install_requires=['msgpack-python>=0.4.6', ]
@@ -64,7 +67,7 @@ except ImportError:
     from distutils.command.build_ext import build_ext
     if not all(os.path.exists(path) for path in [
         compress_source, crypto_source, chunker_source, hashindex_source,
-        platform_linux_source, platform_freebsd_source]):
+        platform_linux_source, platform_freebsd_source]) and not on_rtd:
         raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.')
 
 
@@ -103,10 +106,11 @@ possible_lz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local
 if os.environ.get('BORG_LZ4_PREFIX'):
     possible_openssl_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX'))
 lz4_prefix = detect_lz4(possible_lz4_prefixes)
-if not lz4_prefix:
+if lz4_prefix:
+    include_dirs.append(os.path.join(lz4_prefix, 'include'))
+    library_dirs.append(os.path.join(lz4_prefix, 'lib'))
+elif not on_rtd:
     raise Exception('Unable to find LZ4 headers. (Looked here: {})'.format(', '.join(possible_lz4_prefixes)))
-include_dirs.append(os.path.join(lz4_prefix, 'include'))
-library_dirs.append(os.path.join(lz4_prefix, 'lib'))
 
 
 with open('README.rst', 'r') as fd:
@@ -114,18 +118,20 @@ with open('README.rst', 'r') as fd:
 
 cmdclass = {'build_ext': build_ext, 'sdist': Sdist}
 
-ext_modules = [
+ext_modules = []
+if not on_rtd:
+    ext_modules += [
     Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs),
     Extension('borg.crypto', [crypto_source], libraries=['crypto'], include_dirs=include_dirs, library_dirs=library_dirs),
     Extension('borg.chunker', [chunker_source]),
     Extension('borg.hashindex', [hashindex_source])
 ]
-if sys.platform.startswith('linux'):
-    ext_modules.append(Extension('borg.platform_linux', [platform_linux_source], libraries=['acl']))
-elif sys.platform.startswith('freebsd'):
-    ext_modules.append(Extension('borg.platform_freebsd', [platform_freebsd_source]))
-elif sys.platform == 'darwin':
-    ext_modules.append(Extension('borg.platform_darwin', [platform_darwin_source]))
+    if sys.platform.startswith('linux'):
+        ext_modules.append(Extension('borg.platform_linux', [platform_linux_source], libraries=['acl']))
+    elif sys.platform.startswith('freebsd'):
+        ext_modules.append(Extension('borg.platform_freebsd', [platform_freebsd_source]))
+    elif sys.platform == 'darwin':
+        ext_modules.append(Extension('borg.platform_darwin', [platform_darwin_source]))
 
 setup(
     name='borgbackup',

From 4787424a668d71266063f09d1fa02b1cae310bc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 19:55:56 -0400
Subject: [PATCH 43/70] move API generation to setup.py

---
 docs/Makefile | 17 +----------------
 setup.py      | 40 +++++++++++++++++++++++++++++++++++++++-
 2 files changed, 40 insertions(+), 17 deletions(-)

diff --git a/docs/Makefile b/docs/Makefile
index 133080cdb..1f3f7d76c 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -36,7 +36,7 @@ help:
 clean:
 	-rm -rf $(BUILDDIR)/*
 
-html: usage api.rst
+html: usage
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
@@ -153,18 +153,3 @@ usage/%.rst.inc: ../borg/archiver.py
 	@borg help $* --usage-only | sed -e 's/^/    /' >> $@
 	@printf "\nDescription\n~~~~~~~~~~~\n" >> $@
 	@borg help $* --epilog-only >> $@
-
-api.rst: Makefile
-	@echo "auto-generating API documentation"
-	@echo "Borg Backup API documentation" > $@
-	@echo "=============================" >> $@
-	@echo "" >> $@
-	@for mod in ../borg/*.pyx ../borg/*.py; do \
-		if echo "$$mod" | grep -q "/_"; then \
-			continue ; \
-		fi ; \
-		printf ".. automodule:: "; \
-		echo "$$mod" | sed "s!\.\./!!;s/\.pyx\?//;s!/!.!"; \
-		echo "    :members:"; \
-		echo "    :undoc-members:"; \
-	done >> $@
diff --git a/setup.py b/setup.py
index 2e8ed1f18..ffcbac101 100644
--- a/setup.py
+++ b/setup.py
@@ -3,6 +3,10 @@ import os
 import sys
 from glob import glob
 
+from distutils.command.build import build
+from distutils.core import Command
+from distutils.errors import DistutilsOptionError
+
 min_python = (3, 2)
 my_python = sys.version_info
 
@@ -115,8 +119,42 @@ elif not on_rtd:
 
 with open('README.rst', 'r') as fd:
     long_description = fd.read()
+class build_api(Command):
+    description = "generate a basic api.rst file based on the modules available"
 
-cmdclass = {'build_ext': build_ext, 'sdist': Sdist}
+    user_options = [
+        ('output=', 'O', 'output directory'),
+    ]
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        print("auto-generating API documentation")
+        with open("docs/api.rst", "w") as api:
+            api.write("""
+Borg Backup API documentation"
+=============================
+""")
+            for mod in glob('borg/*.py') + glob('borg/*.pyx'):
+                print("examining module %s" % mod)
+                if "/_" not in mod:
+                    api.write("""
+.. automodule:: %s
+    :members:
+    :undoc-members:
+""" % mod)
+
+# (function, predicate), see http://docs.python.org/2/distutils/apiref.html#distutils.cmd.Command.sub_commands
+build.sub_commands.append(('build_api', None))
+
+cmdclass = {
+    'build_ext': build_ext,
+    'build_api': build_api,
+    'sdist': Sdist
+}
 
 ext_modules = []
 if not on_rtd:

From 13d356854805094e465a14b3d61d62b0875b9285 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 21:07:12 -0400
Subject: [PATCH 44/70] move usage generation to setup.py

this is an unfortunate rewrite of the manpage creation code mentionned
in #208. ideally, this would be rewritten into a class that can
generate both man pages and .rst files.
---
 borg/archiver.py | 50 +++++++++++++++++++++++++----------------------
 docs/Makefile    | 16 +--------------
 setup.py         | 51 ++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 79 insertions(+), 38 deletions(-)

diff --git a/borg/archiver.py b/borg/archiver.py
index 57e30760e..9b18486c8 100644
--- a/borg/archiver.py
+++ b/borg/archiver.py
@@ -548,24 +548,8 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
                     print(warning)
         return args
 
-    def run(self, args=None):
-        check_extension_modules()
-        keys_dir = get_keys_dir()
-        if not os.path.exists(keys_dir):
-            os.makedirs(keys_dir)
-            os.chmod(keys_dir, stat.S_IRWXU)
-        cache_dir = get_cache_dir()
-        if not os.path.exists(cache_dir):
-            os.makedirs(cache_dir)
-            os.chmod(cache_dir, stat.S_IRWXU)
-            with open(os.path.join(cache_dir, 'CACHEDIR.TAG'), 'w') as fd:
-                fd.write(textwrap.dedent("""
-                    Signature: 8a477f597d28d172789f06886806bc55
-                    # This file is a cache directory tag created by Borg.
-                    # For information about cache directory tags, see:
-                    #       http://www.brynosaurus.com/cachedir/
-                    """).lstrip())
-        common_parser = argparse.ArgumentParser(add_help=False)
+    def build_parser(self, args=None, prog=None):
+        common_parser = argparse.ArgumentParser(add_help=False, prog=prog)
         common_parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
                                    default=False,
                                    help='verbose output')
@@ -576,11 +560,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
         common_parser.add_argument('--remote-path', dest='remote_path', default=RemoteRepository.remote_path, metavar='PATH',
                                    help='set remote path to executable (default: "%(default)s")')
 
-        # We can't use argparse for "serve" since we don't want it to show up in "Available commands"
-        if args:
-            args = self.preprocess_args(args)
-
-        parser = argparse.ArgumentParser(description='Borg %s - Deduplicated Backups' % __version__)
+        parser = argparse.ArgumentParser(prog=prog, description='Borg %s - Deduplicated Backups' % __version__)
         subparsers = parser.add_subparsers(title='Available commands')
 
         serve_epilog = textwrap.dedent("""
@@ -976,6 +956,30 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
         subparser.set_defaults(func=functools.partial(self.do_help, parser, subparsers.choices))
         subparser.add_argument('topic', metavar='TOPIC', type=str, nargs='?',
                                help='additional help on TOPIC')
+        return parser
+
+    def run(self, args=None):
+        check_extension_modules()
+        keys_dir = get_keys_dir()
+        if not os.path.exists(keys_dir):
+            os.makedirs(keys_dir)
+            os.chmod(keys_dir, stat.S_IRWXU)
+        cache_dir = get_cache_dir()
+        if not os.path.exists(cache_dir):
+            os.makedirs(cache_dir)
+            os.chmod(cache_dir, stat.S_IRWXU)
+            with open(os.path.join(cache_dir, 'CACHEDIR.TAG'), 'w') as fd:
+                fd.write(textwrap.dedent("""
+                    Signature: 8a477f597d28d172789f06886806bc55
+                    # This file is a cache directory tag created by Borg.
+                    # For information about cache directory tags, see:
+                    #       http://www.brynosaurus.com/cachedir/
+                    """).lstrip())
+
+        # We can't use argparse for "serve" since we don't want it to show up in "Available commands"
+        if args:
+            args = self.preprocess_args(args)
+        parser = self.build_parser(args)
 
         args = parser.parse_args(args or ['-h'])
         self.verbose = args.verbose
diff --git a/docs/Makefile b/docs/Makefile
index 1f3f7d76c..fb470c6a9 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -36,7 +36,7 @@ help:
 clean:
 	-rm -rf $(BUILDDIR)/*
 
-html: usage
+html:
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
@@ -139,17 +139,3 @@ gh-io: html
 
 inotify: html
 	while inotifywait -r . --exclude usage.rst --exclude '_build/*' ; do make html ; done
-
-# generate list of targets
-usage: $(shell borg help | grep -A1 "Available commands:" | tail -1 | sed 's/[{} ]//g;s/,\|^/.rst.inc usage\//g;s/^.rst.inc//;s/usage\/help//')
-
-# generate help file based on usage
-usage/%.rst.inc: ../borg/archiver.py
-	@echo generating usage for $*
-	@printf ".. _borg_$*:\n\n" > $@
-	@printf "borg $*\n" >> $@
-	@echo -n borg $* | tr 'a-z- ' '-' >> $@
-	@printf "\n::\n\n" >> $@
-	@borg help $* --usage-only | sed -e 's/^/    /' >> $@
-	@printf "\nDescription\n~~~~~~~~~~~\n" >> $@
-	@borg help $* --epilog-only >> $@
diff --git a/setup.py b/setup.py
index ffcbac101..2f29b4a20 100644
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,6 @@
 # -*- encoding: utf-8 *-*
 import os
+import re
 import sys
 from glob import glob
 
@@ -119,6 +120,54 @@ elif not on_rtd:
 
 with open('README.rst', 'r') as fd:
     long_description = fd.read()
+
+class build_usage(Command):
+    description = "generate usage for each command"
+
+    user_options = [
+        ('output=', 'O', 'output directory'),
+    ]
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        import pdb
+        print('generating usage docs')
+        from borg.archiver import Archiver
+        parser = Archiver().build_parser(prog='borg')
+        choices = {}
+        for action in parser._actions:
+            if action.choices is not None:
+                choices.update(action.choices)
+        print('found commands: %s' % list(choices.keys()))
+        if not os.path.exists('docs/usage'):
+            os.mkdir('docs/usage')
+        for command, parser in choices.items():
+            if command is 'help':
+                continue
+            with open('docs/usage/%s.rst.inc' % command, 'w') as cmdfile:
+                print('generating help for %s' % command)
+                cmdfile.write(""".. _borg_{command}:
+
+borg {command}
+{underline}
+::
+
+""".format(**{"command": command,
+                              "underline": '-' * len('borg ' + command)}))
+                epilog = parser.epilog
+                parser.epilog = None
+                cmdfile.write(re.sub("^", "    ", parser.format_help(), flags=re.M))
+                cmdfile.write("""
+Description
+~~~~~~~~~~~
+""")
+                cmdfile.write(epilog)
+
+
 class build_api(Command):
     description = "generate a basic api.rst file based on the modules available"
 
@@ -149,10 +198,12 @@ Borg Backup API documentation"
 
 # (function, predicate), see http://docs.python.org/2/distutils/apiref.html#distutils.cmd.Command.sub_commands
 build.sub_commands.append(('build_api', None))
+build.sub_commands.append(('build_usage', None))
 
 cmdclass = {
     'build_ext': build_ext,
     'build_api': build_api,
+    'build_usage': build_usage,
     'sdist': Sdist
 }
 

From 8fe56f001cfeeb027162b38da14cfec9f6038754 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 21:10:09 -0400
Subject: [PATCH 45/70] main website becomes RTD, main website link is
 pointless

---
 docs/_themes/local/sidebarusefullinks.html | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/_themes/local/sidebarusefullinks.html b/docs/_themes/local/sidebarusefullinks.html
index a311fe558..a323ebe69 100644
--- a/docs/_themes/local/sidebarusefullinks.html
+++ b/docs/_themes/local/sidebarusefullinks.html
@@ -3,7 +3,6 @@
 
 <h3>Useful Links</h3>
 <ul>
-  <li><a href="https://borgbackup.github.io/borgbackup/">Main Web Site</a></li>
   <li><a href="https://github.com/borgbackup/borg/releases">Releases</a></li>
   <li><a href="https://pypi.python.org/pypi/borgbackup">PyPI packages</a></li>
   <li><a href="https://github.com/borgbackup/borg/blob/master/CHANGES.rst">Current ChangeLog</a></li>

From 9cbc868764a83fe6e1d39e07c5dfdef3e8baa72a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 22:24:30 -0400
Subject: [PATCH 46/70] make tests and build work again in usage using
 environment

this is such a crude hack it is totally embarrassing....

the proper solution would probably be to move the `build_parser()`
function out of `Archiver` completely, but this is such an undertaking
that i doubt it is worth doing since we're looking at switching to
click anyways.

the main problem in moving build_parser() out is that it references
`self` all the time, so it *needs* an archiver context that it can
reuse. we could make the function static and pass self in there by
hand, but it seems like almost a worse hack... and besides, we would
need to load the archiver in order to do that, which would break usage
all over again...
---
 borg/archive.py    |  7 ++++---
 borg/archiver.py   | 11 ++++++-----
 borg/helpers.py    |  7 ++++---
 borg/key.py        |  5 +++--
 borg/repository.py |  3 ++-
 setup.py           |  2 ++
 6 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/borg/archive.py b/borg/archive.py
index d6eff1ba9..8350bbebc 100644
--- a/borg/archive.py
+++ b/borg/archive.py
@@ -12,9 +12,10 @@ import sys
 import time
 from io import BytesIO
 from . import xattr
-from .platform import acl_get, acl_set
-from .chunker import Chunker
-from .hashindex import ChunkIndex
+if not os.environ.get('BORG_GEN_USAGE', False):
+    from .platform import acl_get, acl_set
+    from .chunker import Chunker
+    from .hashindex import ChunkIndex
 from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \
     Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int
 
diff --git a/borg/archiver.py b/borg/archiver.py
index 9b18486c8..bd49da7c9 100644
--- a/borg/archiver.py
+++ b/borg/archiver.py
@@ -15,12 +15,13 @@ import textwrap
 import traceback
 
 from . import __version__
+if not os.environ.get('BORG_GEN_USAGE', False):
+    from .compress import Compressor, COMPR_BUFFER
+    from .upgrader import AtticRepositoryUpgrader
+    from .repository import Repository
+    from .cache import Cache
+    from .key import key_creator
 from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS
-from .compress import Compressor, COMPR_BUFFER
-from .upgrader import AtticRepositoryUpgrader
-from .repository import Repository
-from .cache import Cache
-from .key import key_creator
 from .helpers import Error, location_validator, format_time, format_file_size, \
     format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \
     get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \
diff --git a/borg/helpers.py b/borg/helpers.py
index 47d454bec..175ebf15c 100644
--- a/borg/helpers.py
+++ b/borg/helpers.py
@@ -18,9 +18,10 @@ from operator import attrgetter
 
 import msgpack
 
-from . import hashindex
-from . import chunker
-from . import crypto
+if not os.environ.get('BORG_GEN_USAGE', False):
+    from . import hashindex
+    from . import chunker
+    from . import crypto
 
 
 class Error(Exception):
diff --git a/borg/key.py b/borg/key.py
index 7067a4454..877b37711 100644
--- a/borg/key.py
+++ b/borg/key.py
@@ -7,8 +7,9 @@ import textwrap
 import hmac
 from hashlib import sha256
 
-from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
-from .compress import Compressor, COMPR_BUFFER
+if not os.environ.get('BORG_GEN_USAGE', False):
+    from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
+    from .compress import Compressor, COMPR_BUFFER
 from .helpers import IntegrityError, get_keys_dir, Error
 
 PREFIX = b'\0' * 8
diff --git a/borg/repository.py b/borg/repository.py
index 932e4fef3..30163b02c 100644
--- a/borg/repository.py
+++ b/borg/repository.py
@@ -8,7 +8,8 @@ import struct
 import sys
 from zlib import crc32
 
-from .hashindex import NSIndex
+if not os.environ.get('BORG_GEN_USAGE', False):
+    from .hashindex import NSIndex
 from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify
 from .locking import UpgradableLock
 from .lrucache import LRUCache
diff --git a/setup.py b/setup.py
index 2f29b4a20..3a5286b70 100644
--- a/setup.py
+++ b/setup.py
@@ -136,6 +136,8 @@ class build_usage(Command):
     def run(self):
         import pdb
         print('generating usage docs')
+        # XXX: gross hack: allows us to skip loading C modules during help generation
+        os.environ['BORG_GEN_USAGE'] = "True"
         from borg.archiver import Archiver
         parser = Archiver().build_parser(prog='borg')
         choices = {}

From 712170c71a833e1f0922a207058b9b64ce40852a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Wed, 7 Oct 2015 22:39:18 -0400
Subject: [PATCH 47/70] fix virtualenv path in .gitignore

the current instructions create a dirty tree that `git add .` would
commit into git. this is error prone and somewhat unclean.

i found it preferable to change the `.gitignore` than to change the instructions, since there are probably `borg-env` environments lying around everywhere already.
---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 4f7c67672..860c8aeb5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,7 +2,7 @@ MANIFEST
 docs/_build
 build
 dist
-env
+borg-env
 .tox
 hashindex.c
 chunker.c

From c93d975d3f5aa240899dfdba4fa4a83b6f8c5049 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 08:20:59 -0400
Subject: [PATCH 48/70] remove debugging code

---
 setup.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/setup.py b/setup.py
index 3a5286b70..1b411c355 100644
--- a/setup.py
+++ b/setup.py
@@ -134,7 +134,6 @@ class build_usage(Command):
         pass
 
     def run(self):
-        import pdb
         print('generating usage docs')
         # XXX: gross hack: allows us to skip loading C modules during help generation
         os.environ['BORG_GEN_USAGE'] = "True"

From 45d9c6b3b7b6b23c0fe292798d365050842f32fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 08:21:52 -0400
Subject: [PATCH 49/70] fix formatting

---
 setup.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/setup.py b/setup.py
index 1b411c355..955d5d758 100644
--- a/setup.py
+++ b/setup.py
@@ -151,21 +151,14 @@ class build_usage(Command):
                 continue
             with open('docs/usage/%s.rst.inc' % command, 'w') as cmdfile:
                 print('generating help for %s' % command)
-                cmdfile.write(""".. _borg_{command}:
-
-borg {command}
-{underline}
-::
-
-""".format(**{"command": command,
-                              "underline": '-' * len('borg ' + command)}))
+                params = {"command": command,
+                          "underline": '-' * len('borg ' + command)}
+                cmdfile.write(".. _borg_{command}:\n\n".format(**params))
+                cmdfile.write("borg {command}\n{underline}\n::\n\n".format(**params))
                 epilog = parser.epilog
                 parser.epilog = None
                 cmdfile.write(re.sub("^", "    ", parser.format_help(), flags=re.M))
-                cmdfile.write("""
-Description
-~~~~~~~~~~~
-""")
+                cmdfile.write("\nDescription\n~~~~~~~~~~~\n")
                 cmdfile.write(epilog)
 
 

From 8190e33a64790a42d545e60cf9298e1c247cd99d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 08:22:52 -0400
Subject: [PATCH 50/70] faster and cleaner rtd check for cython

---
 setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 955d5d758..dcf486c1e 100644
--- a/setup.py
+++ b/setup.py
@@ -70,9 +70,9 @@ except ImportError:
     platform_freebsd_source = platform_freebsd_source.replace('.pyx', '.c')
     platform_darwin_source = platform_darwin_source.replace('.pyx', '.c')
     from distutils.command.build_ext import build_ext
-    if not all(os.path.exists(path) for path in [
+    if not on_rtd and not all(os.path.exists(path) for path in [
         compress_source, crypto_source, chunker_source, hashindex_source,
-        platform_linux_source, platform_freebsd_source]) and not on_rtd:
+        platform_linux_source, platform_freebsd_source]):
         raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.')
 
 

From 7ccf6b32a6c8770c497cf77a376f60b4ef6ed8cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 08:23:37 -0400
Subject: [PATCH 51/70] simplify RTD env check

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index dcf486c1e..b691ac2e6 100644
--- a/setup.py
+++ b/setup.py
@@ -16,7 +16,7 @@ if my_python < min_python:
     sys.exit(1)
 
 # Are we building on ReadTheDocs?
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+on_rtd = os.environ.get('READTHEDOCS')
 
 # msgpack pure python data corruption was fixed in 0.4.6.
 # Also, we might use some rather recent API features.

From da02f373c7e6f136106e652b54672b6a54e847d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 08:24:46 -0400
Subject: [PATCH 52/70] Revert "main website becomes RTD, main website link is
 pointless"

Instead, we put the readthedocs.org site as main website.

This reverts commit 8fe56f001cfeeb027162b38da14cfec9f6038754.
---
 docs/_themes/local/sidebarusefullinks.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/_themes/local/sidebarusefullinks.html b/docs/_themes/local/sidebarusefullinks.html
index a323ebe69..c9ae280b2 100644
--- a/docs/_themes/local/sidebarusefullinks.html
+++ b/docs/_themes/local/sidebarusefullinks.html
@@ -3,6 +3,7 @@
 
 <h3>Useful Links</h3>
 <ul>
+  <li><a href="https://borgbackup.readthedocs.org/">Main Web Site</a></li>
   <li><a href="https://github.com/borgbackup/borg/releases">Releases</a></li>
   <li><a href="https://pypi.python.org/pypi/borgbackup">PyPI packages</a></li>
   <li><a href="https://github.com/borgbackup/borg/blob/master/CHANGES.rst">Current ChangeLog</a></li>

From 6f9e04bc2101936ed69bbd1f8fffe94a74db0d6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 08:29:02 -0400
Subject: [PATCH 53/70] generalise the cython check hack

instead of applying this only to usage generation, use it as a generic
mechanism to disable loading of Cython code.

it may be incomplete: there may be other places where Cython code is
loaded that is not checked, but that is sufficient to build the usage
docs. the environment variable used is documented as such in the
docs/usage.rst.

we also move the check to a helper function and document it
better. this has the unfortunate side effect of moving includes
around, but I can't think of a better way.
---
 borg/archive.py    |  6 +++---
 borg/archiver.py   | 12 ++++++------
 borg/helpers.py    | 15 ++++++++++++++-
 borg/key.py        |  4 ++--
 borg/repository.py |  4 ++--
 docs/usage.rst     |  6 ++++++
 setup.py           |  8 ++++++--
 7 files changed, 39 insertions(+), 16 deletions(-)

diff --git a/borg/archive.py b/borg/archive.py
index 8350bbebc..43fd6e66c 100644
--- a/borg/archive.py
+++ b/borg/archive.py
@@ -12,12 +12,12 @@ import sys
 import time
 from io import BytesIO
 from . import xattr
-if not os.environ.get('BORG_GEN_USAGE', False):
+from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \
+    Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int, detect_cython
+if not detect_cython():
     from .platform import acl_get, acl_set
     from .chunker import Chunker
     from .hashindex import ChunkIndex
-from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \
-    Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int
 
 ITEMS_BUFFER = 1024 * 1024
 
diff --git a/borg/archiver.py b/borg/archiver.py
index bd49da7c9..4b831f2e0 100644
--- a/borg/archiver.py
+++ b/borg/archiver.py
@@ -15,18 +15,18 @@ import textwrap
 import traceback
 
 from . import __version__
-if not os.environ.get('BORG_GEN_USAGE', False):
+from .helpers import Error, location_validator, format_time, format_file_size, \
+    format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \
+    get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \
+    Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
+    is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, detect_cython
+if not detect_cython():
     from .compress import Compressor, COMPR_BUFFER
     from .upgrader import AtticRepositoryUpgrader
     from .repository import Repository
     from .cache import Cache
     from .key import key_creator
 from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS
-from .helpers import Error, location_validator, format_time, format_file_size, \
-    format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \
-    get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \
-    Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
-    is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec
 from .remote import RepositoryServer, RemoteRepository
 
 has_lchflags = hasattr(os, 'lchflags')
diff --git a/borg/helpers.py b/borg/helpers.py
index 175ebf15c..250f899af 100644
--- a/borg/helpers.py
+++ b/borg/helpers.py
@@ -18,7 +18,20 @@ from operator import attrgetter
 
 import msgpack
 
-if not os.environ.get('BORG_GEN_USAGE', False):
+def detect_cython():
+    """allow for a way to disable Cython includes
+
+    this is used during usage docs build, in setup.py. It is to avoid
+    loading the Cython libraries which are built, but sometimes not in
+    the search path (namely, during Tox runs).
+
+    we simply check an environment variable (``BORG_CYTHON_DISABLE``)
+    which, when set (to anything) will disable includes of Cython
+    libraries in key places to enable usage docs to be built.
+    """
+    return os.environ.get('BORG_CYTHON_DISABLE')
+
+if not detect_cython():
     from . import hashindex
     from . import chunker
     from . import crypto
diff --git a/borg/key.py b/borg/key.py
index 877b37711..ace5c37e8 100644
--- a/borg/key.py
+++ b/borg/key.py
@@ -7,10 +7,10 @@ import textwrap
 import hmac
 from hashlib import sha256
 
-if not os.environ.get('BORG_GEN_USAGE', False):
+from .helpers import IntegrityError, get_keys_dir, Error, detect_cython
+if not detect_cython():
     from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
     from .compress import Compressor, COMPR_BUFFER
-from .helpers import IntegrityError, get_keys_dir, Error
 
 PREFIX = b'\0' * 8
 
diff --git a/borg/repository.py b/borg/repository.py
index 30163b02c..80a8adbf9 100644
--- a/borg/repository.py
+++ b/borg/repository.py
@@ -8,9 +8,9 @@ import struct
 import sys
 from zlib import crc32
 
-if not os.environ.get('BORG_GEN_USAGE', False):
+from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify, detect_cython
+if not detect_cython():
     from .hashindex import NSIndex
-from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify
 from .locking import UpgradableLock
 from .lrucache import LRUCache
 
diff --git a/docs/usage.rst b/docs/usage.rst
index 6bd292e14..fb7768528 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -60,6 +60,12 @@ Some "yes" sayers (if set, they automatically confirm that you really want to do
         For "Warning: The repository at location ... was previously located at ..."
     BORG_CHECK_I_KNOW_WHAT_I_AM_DOING
         For "Warning: 'check --repair' is an experimental feature that might result in data loss."
+    BORG_CYTHON_DISABLE
+        Disables the loading of Cython modules. This is currently
+        experimentaly and is used only to generate usage docs at build
+        time, it's unlikely to produce good results on a regular
+        run. The variable should be set to the calling class, and
+        should be unique. It is currently only used by ``build_usage``.
 
 Directories:
     BORG_KEYS_DIR
diff --git a/setup.py b/setup.py
index b691ac2e6..67fe0c738 100644
--- a/setup.py
+++ b/setup.py
@@ -135,10 +135,14 @@ class build_usage(Command):
 
     def run(self):
         print('generating usage docs')
-        # XXX: gross hack: allows us to skip loading C modules during help generation
-        os.environ['BORG_GEN_USAGE'] = "True"
+        # allows us to build docs without the C modules fully loaded during help generation
+        if 'BORG_CYTHON_DISABLE' not in os.environ:
+            os.environ['BORG_CYTHON_DISABLE'] = self.__class__.__name__
         from borg.archiver import Archiver
         parser = Archiver().build_parser(prog='borg')
+        # return to regular Cython configuration, if we changed it
+        if os.environ.get('BORG_CYTHON_DISABLE') == self.__class__.__name__:
+            del os.environ['BORG_CYTHON_DISABLE']
         choices = {}
         for action in parser._actions:
             if action.choices is not None:

From 824f9c72a2c6d7e09f5ddba7747d7d963b5bebdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 08:54:49 -0400
Subject: [PATCH 54/70] cosmetic: s/cmdfile/doc/

---
 setup.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/setup.py b/setup.py
index 67fe0c738..c3e6ad887 100644
--- a/setup.py
+++ b/setup.py
@@ -153,17 +153,17 @@ class build_usage(Command):
         for command, parser in choices.items():
             if command is 'help':
                 continue
-            with open('docs/usage/%s.rst.inc' % command, 'w') as cmdfile:
+            with open('docs/usage/%s.rst.inc' % command, 'w') as doc:
                 print('generating help for %s' % command)
                 params = {"command": command,
                           "underline": '-' * len('borg ' + command)}
-                cmdfile.write(".. _borg_{command}:\n\n".format(**params))
-                cmdfile.write("borg {command}\n{underline}\n::\n\n".format(**params))
+                doc.write(".. _borg_{command}:\n\n".format(**params))
+                doc.write("borg {command}\n{underline}\n::\n\n".format(**params))
                 epilog = parser.epilog
                 parser.epilog = None
-                cmdfile.write(re.sub("^", "    ", parser.format_help(), flags=re.M))
-                cmdfile.write("\nDescription\n~~~~~~~~~~~\n")
-                cmdfile.write(epilog)
+                doc.write(re.sub("^", "    ", parser.format_help(), flags=re.M))
+                doc.write("\nDescription\n~~~~~~~~~~~\n")
+                doc.write(epilog)
 
 
 class build_api(Command):

From 86487d192a9a5ab7ff4eedb92d793485b4c30268 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 10:26:02 -0400
Subject: [PATCH 55/70] use build_py to fix build on RTD

it seems that our subcommands are ignored over there, for some mysterious reason.
---
 setup.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/setup.py b/setup.py
index c3e6ad887..7376c56e2 100644
--- a/setup.py
+++ b/setup.py
@@ -7,6 +7,8 @@ from glob import glob
 from distutils.command.build import build
 from distutils.core import Command
 from distutils.errors import DistutilsOptionError
+from distutils import log
+from setuptools.command.build_py import build_py
 
 min_python = (3, 2)
 my_python = sys.version_info
@@ -195,13 +197,38 @@ Borg Backup API documentation"
 """ % mod)
 
 # (function, predicate), see http://docs.python.org/2/distutils/apiref.html#distutils.cmd.Command.sub_commands
+# seems like this doesn't work on RTD, see below for build_py hack.
 build.sub_commands.append(('build_api', None))
 build.sub_commands.append(('build_usage', None))
 
+
+class build_py_custom(build_py):
+    """override build_py to also build our stuf
+
+    it is unclear why this is necessary, but in some environments
+    (Readthedocs.org, specifically), the above
+    ``build.sub_commands.append()`` doesn't seem to have an effect:
+    our custom build commands seem to be ignored when running
+    ``setup.py install``.
+
+    This class overrides the ``build_py`` target by forcing it to run
+    our custom steps as well.
+
+    See also the `bug report on RTD
+    <https://github.com/rtfd/readthedocs.org/issues/1740>`_.
+    """
+    def run(self):
+        super().run()
+        self.announce('calling custom build steps', level=log.INFO)
+        self.run_command('build_api')
+        self.run_command('build_usage')
+
+
 cmdclass = {
     'build_ext': build_ext,
     'build_api': build_api,
     'build_usage': build_usage,
+    'build_py': build_py_custom,
     'sdist': Sdist
 }
 

From 6c5a7733a222c0da1d0950593ba23bdc9b2579f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 10:37:19 -0400
Subject: [PATCH 56/70] force build-ext build

it seems that what worked in the debug branch is not working in the
main branch, even though the commit IDs are exactly the same. the RTD environment doesn't seem really reliable...

besides, we want to build extensions before the rest, so should run it
first, in order to have msgpack loaded.
---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index 7376c56e2..8f31a927f 100644
--- a/setup.py
+++ b/setup.py
@@ -220,6 +220,7 @@ class build_py_custom(build_py):
     def run(self):
         super().run()
         self.announce('calling custom build steps', level=log.INFO)
+        self.run_command('build_ext')
         self.run_command('build_api')
         self.run_command('build_usage')
 

From 2e9bd5a8832364d067ecf11bc6d26dbcf2701113 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 14:57:32 -0400
Subject: [PATCH 57/70] cosmetic: s/api/doc/

---
 setup.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index 8f31a927f..e3338280c 100644
--- a/setup.py
+++ b/setup.py
@@ -182,15 +182,15 @@ class build_api(Command):
 
     def run(self):
         print("auto-generating API documentation")
-        with open("docs/api.rst", "w") as api:
-            api.write("""
+        with open("docs/api.rst", "w") as doc:
+            doc.write("""
 Borg Backup API documentation"
 =============================
 """)
             for mod in glob('borg/*.py') + glob('borg/*.pyx'):
                 print("examining module %s" % mod)
                 if "/_" not in mod:
-                    api.write("""
+                    doc.write("""
 .. automodule:: %s
     :members:
     :undoc-members:

From f98998f042200d497de75f156c434f3fea216b30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 15:26:40 -0400
Subject: [PATCH 58/70] fix logical inversion in the semantics of
 detect_cython()

---
 borg/archive.py    | 2 +-
 borg/archiver.py   | 2 +-
 borg/helpers.py    | 4 ++--
 borg/key.py        | 2 +-
 borg/repository.py | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/borg/archive.py b/borg/archive.py
index 43fd6e66c..c50759bc6 100644
--- a/borg/archive.py
+++ b/borg/archive.py
@@ -14,7 +14,7 @@ from io import BytesIO
 from . import xattr
 from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \
     Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int, detect_cython
-if not detect_cython():
+if detect_cython():
     from .platform import acl_get, acl_set
     from .chunker import Chunker
     from .hashindex import ChunkIndex
diff --git a/borg/archiver.py b/borg/archiver.py
index 4b831f2e0..dca7e4af7 100644
--- a/borg/archiver.py
+++ b/borg/archiver.py
@@ -20,7 +20,7 @@ from .helpers import Error, location_validator, format_time, format_file_size, \
     get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \
     Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
     is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, detect_cython
-if not detect_cython():
+if detect_cython():
     from .compress import Compressor, COMPR_BUFFER
     from .upgrader import AtticRepositoryUpgrader
     from .repository import Repository
diff --git a/borg/helpers.py b/borg/helpers.py
index 250f899af..a57feda96 100644
--- a/borg/helpers.py
+++ b/borg/helpers.py
@@ -29,9 +29,9 @@ def detect_cython():
     which, when set (to anything) will disable includes of Cython
     libraries in key places to enable usage docs to be built.
     """
-    return os.environ.get('BORG_CYTHON_DISABLE')
+    return not os.environ.get('BORG_CYTHON_DISABLE')
 
-if not detect_cython():
+if detect_cython():
     from . import hashindex
     from . import chunker
     from . import crypto
diff --git a/borg/key.py b/borg/key.py
index ace5c37e8..2d0ffc86e 100644
--- a/borg/key.py
+++ b/borg/key.py
@@ -8,7 +8,7 @@ import hmac
 from hashlib import sha256
 
 from .helpers import IntegrityError, get_keys_dir, Error, detect_cython
-if not detect_cython():
+if detect_cython():
     from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
     from .compress import Compressor, COMPR_BUFFER
 
diff --git a/borg/repository.py b/borg/repository.py
index 80a8adbf9..762a40302 100644
--- a/borg/repository.py
+++ b/borg/repository.py
@@ -9,7 +9,7 @@ import sys
 from zlib import crc32
 
 from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify, detect_cython
-if not detect_cython():
+if detect_cython():
     from .hashindex import NSIndex
 from .locking import UpgradableLock
 from .lrucache import LRUCache

From ff483fe48552389a9e898ba9da2ccb2e7b3e26f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 15:26:40 -0400
Subject: [PATCH 59/70] fix logical inversion in the semantics of
 detect_cython()

---
 borg/helpers.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/borg/helpers.py b/borg/helpers.py
index a57feda96..bfbc45a21 100644
--- a/borg/helpers.py
+++ b/borg/helpers.py
@@ -28,6 +28,8 @@ def detect_cython():
     we simply check an environment variable (``BORG_CYTHON_DISABLE``)
     which, when set (to anything) will disable includes of Cython
     libraries in key places to enable usage docs to be built.
+
+    :returns: True if Cython is available, False otherwise.
     """
     return not os.environ.get('BORG_CYTHON_DISABLE')
 

From 2f803b6489278913a270449c2d1b1f90cfa953c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 15:28:53 -0400
Subject: [PATCH 60/70] fix typo, split sentence

---
 docs/usage.rst | 4 ++--
 setup.py       | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/usage.rst b/docs/usage.rst
index fb7768528..b316f17d9 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -62,8 +62,8 @@ Some "yes" sayers (if set, they automatically confirm that you really want to do
         For "Warning: 'check --repair' is an experimental feature that might result in data loss."
     BORG_CYTHON_DISABLE
         Disables the loading of Cython modules. This is currently
-        experimentaly and is used only to generate usage docs at build
-        time, it's unlikely to produce good results on a regular
+        experimental and is used only to generate usage docs at build
+        time. It is unlikely to produce good results on a regular
         run. The variable should be set to the calling class, and
         should be unique. It is currently only used by ``build_usage``.
 
diff --git a/setup.py b/setup.py
index e3338280c..f1bb8f68b 100644
--- a/setup.py
+++ b/setup.py
@@ -203,7 +203,7 @@ build.sub_commands.append(('build_usage', None))
 
 
 class build_py_custom(build_py):
-    """override build_py to also build our stuf
+    """override build_py to also build our stuff
 
     it is unclear why this is necessary, but in some environments
     (Readthedocs.org, specifically), the above

From 7ba4d47f6e849dc6a3ebd836afa29bc44bb339ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 15:29:50 -0400
Subject: [PATCH 61/70] clarify the class name part

---
 docs/usage.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/usage.rst b/docs/usage.rst
index b316f17d9..80f3eaa2f 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -64,8 +64,8 @@ Some "yes" sayers (if set, they automatically confirm that you really want to do
         Disables the loading of Cython modules. This is currently
         experimental and is used only to generate usage docs at build
         time. It is unlikely to produce good results on a regular
-        run. The variable should be set to the calling class, and
-        should be unique. It is currently only used by ``build_usage``.
+        run. The variable should be set to the name of the  calling class, and
+        should be unique across all of borg. It is currently only used by ``build_usage``.
 
 Directories:
     BORG_KEYS_DIR

From 423ff45d816e9608edd8e800db24ea8522e50802 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 15:34:44 -0400
Subject: [PATCH 62/70] rename cython detection function

"have" has clearer semantics than "detect"
---
 borg/archive.py    | 4 ++--
 borg/archiver.py   | 4 ++--
 borg/helpers.py    | 4 ++--
 borg/key.py        | 4 ++--
 borg/repository.py | 4 ++--
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/borg/archive.py b/borg/archive.py
index c50759bc6..27cd9d0aa 100644
--- a/borg/archive.py
+++ b/borg/archive.py
@@ -13,8 +13,8 @@ import time
 from io import BytesIO
 from . import xattr
 from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \
-    Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int, detect_cython
-if detect_cython():
+    Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int, have_cython
+if have_cython():
     from .platform import acl_get, acl_set
     from .chunker import Chunker
     from .hashindex import ChunkIndex
diff --git a/borg/archiver.py b/borg/archiver.py
index dca7e4af7..b30a47b88 100644
--- a/borg/archiver.py
+++ b/borg/archiver.py
@@ -19,8 +19,8 @@ from .helpers import Error, location_validator, format_time, format_file_size, \
     format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \
     get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \
     Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
-    is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, detect_cython
-if detect_cython():
+    is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, have_cython
+if have_cython():
     from .compress import Compressor, COMPR_BUFFER
     from .upgrader import AtticRepositoryUpgrader
     from .repository import Repository
diff --git a/borg/helpers.py b/borg/helpers.py
index bfbc45a21..10ee0441a 100644
--- a/borg/helpers.py
+++ b/borg/helpers.py
@@ -18,7 +18,7 @@ from operator import attrgetter
 
 import msgpack
 
-def detect_cython():
+def have_cython():
     """allow for a way to disable Cython includes
 
     this is used during usage docs build, in setup.py. It is to avoid
@@ -33,7 +33,7 @@ def detect_cython():
     """
     return not os.environ.get('BORG_CYTHON_DISABLE')
 
-if detect_cython():
+if have_cython():
     from . import hashindex
     from . import chunker
     from . import crypto
diff --git a/borg/key.py b/borg/key.py
index 2d0ffc86e..81f9185c8 100644
--- a/borg/key.py
+++ b/borg/key.py
@@ -7,8 +7,8 @@ import textwrap
 import hmac
 from hashlib import sha256
 
-from .helpers import IntegrityError, get_keys_dir, Error, detect_cython
-if detect_cython():
+from .helpers import IntegrityError, get_keys_dir, Error, have_cython
+if have_cython():
     from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
     from .compress import Compressor, COMPR_BUFFER
 
diff --git a/borg/repository.py b/borg/repository.py
index 762a40302..c762d09dc 100644
--- a/borg/repository.py
+++ b/borg/repository.py
@@ -8,8 +8,8 @@ import struct
 import sys
 from zlib import crc32
 
-from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify, detect_cython
-if detect_cython():
+from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify, have_cython
+if have_cython():
     from .hashindex import NSIndex
 from .locking import UpgradableLock
 from .lrucache import LRUCache

From f2c56fb890a3a8426708e20c95a7a76b8ef87968 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 16:57:36 -0400
Subject: [PATCH 63/70] try to fix build on RTD *again*

---
 borg/helpers.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/borg/helpers.py b/borg/helpers.py
index 10ee0441a..a0eb0cb9e 100644
--- a/borg/helpers.py
+++ b/borg/helpers.py
@@ -16,8 +16,6 @@ from datetime import datetime, timezone, timedelta
 from fnmatch import translate
 from operator import attrgetter
 
-import msgpack
-
 def have_cython():
     """allow for a way to disable Cython includes
 
@@ -37,6 +35,7 @@ if have_cython():
     from . import hashindex
     from . import chunker
     from . import crypto
+    import msgpack
 
 
 class Error(Exception):

From e8ae96b54e637f5a5326bc296bad0cf22bb87592 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 17:01:42 -0400
Subject: [PATCH 64/70] try to fix RTD build *again*

---
 borg/archive.py | 3 ++-
 borg/key.py     | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/borg/archive.py b/borg/archive.py
index 27cd9d0aa..77638d6c1 100644
--- a/borg/archive.py
+++ b/borg/archive.py
@@ -4,7 +4,7 @@ from itertools import groupby
 import errno
 from .key import key_factory
 from .remote import cache_if_remote
-import msgpack
+
 import os
 import socket
 import stat
@@ -18,6 +18,7 @@ if have_cython():
     from .platform import acl_get, acl_set
     from .chunker import Chunker
     from .hashindex import ChunkIndex
+    import msgpack
 
 ITEMS_BUFFER = 1024 * 1024
 
diff --git a/borg/key.py b/borg/key.py
index 81f9185c8..3e8150f2a 100644
--- a/borg/key.py
+++ b/borg/key.py
@@ -2,7 +2,6 @@ from binascii import hexlify, a2b_base64, b2a_base64
 import configparser
 import getpass
 import os
-import msgpack
 import textwrap
 import hmac
 from hashlib import sha256
@@ -11,6 +10,7 @@ from .helpers import IntegrityError, get_keys_dir, Error, have_cython
 if have_cython():
     from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
     from .compress import Compressor, COMPR_BUFFER
+    import msgpack
 
 PREFIX = b'\0' * 8
 

From a869ab0702e266d6876482cf86566d3bd633d0a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 17:03:35 -0400
Subject: [PATCH 65/70] try to fix RTD build *again* *again*

---
 borg/cache.py  | 6 ++++--
 borg/fuse.py   | 6 ++++--
 borg/remote.py | 6 ++++--
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/borg/cache.py b/borg/cache.py
index e034c8400..0f559f15e 100644
--- a/borg/cache.py
+++ b/borg/cache.py
@@ -1,7 +1,6 @@
 import configparser
 from .remote import cache_if_remote
 import errno
-import msgpack
 import os
 import stat
 import sys
@@ -12,10 +11,13 @@ import tempfile
 
 from .key import PlaintextKey
 from .helpers import Error, get_cache_dir, decode_dict, st_mtime_ns, unhexlify, int_to_bigint, \
-    bigint_to_int
+    bigint_to_int, have_cython
 from .locking import UpgradableLock
 from .hashindex import ChunkIndex
 
+if have_cython:
+    import msgpack
+
 
 class Cache:
     """Client Side cache
diff --git a/borg/fuse.py b/borg/fuse.py
index a54a417b8..d7a5aa5ce 100644
--- a/borg/fuse.py
+++ b/borg/fuse.py
@@ -2,15 +2,17 @@ from collections import defaultdict
 import errno
 import io
 import llfuse
-import msgpack
 import os
 import stat
 import tempfile
 import time
 from .archive import Archive
-from .helpers import daemonize
+from .helpers import daemonize, have_cython
 from .remote import cache_if_remote
 
+if have_cython:
+    import msgpack
+
 # Does this version of llfuse support ns precision?
 have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns')
 
diff --git a/borg/remote.py b/borg/remote.py
index b9847c7e4..466d627da 100644
--- a/borg/remote.py
+++ b/borg/remote.py
@@ -1,6 +1,5 @@
 import errno
 import fcntl
-import msgpack
 import os
 import select
 import shlex
@@ -11,9 +10,12 @@ import traceback
 
 from . import __version__
 
-from .helpers import Error, IntegrityError
+from .helpers import Error, IntegrityError, have_cython
 from .repository import Repository
 
+if have_cython:
+    import msgpack
+
 BUFSIZE = 10 * 1024 * 1024
 
 

From a1dad8c9dac86483c6ba04d779e9d80f9f7b2504 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 17:06:48 -0400
Subject: [PATCH 66/70] try to mock msgpack altogether to fix RTD again

it seems that msgpack is a hard depends in archive...
---
 borg/archive.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/borg/archive.py b/borg/archive.py
index 77638d6c1..f10d24d92 100644
--- a/borg/archive.py
+++ b/borg/archive.py
@@ -19,6 +19,9 @@ if have_cython():
     from .chunker import Chunker
     from .hashindex import ChunkIndex
     import msgpack
+else:
+    import mock
+    msgpack = mock.Mock()
 
 ITEMS_BUFFER = 1024 * 1024
 

From d6109b676cfcf8c3742da6ef1973e8d8df772c62 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 17:09:33 -0400
Subject: [PATCH 67/70] re-enable cython as late as possible

---
 setup.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index f1bb8f68b..f2a3f8bb9 100644
--- a/setup.py
+++ b/setup.py
@@ -142,9 +142,6 @@ class build_usage(Command):
             os.environ['BORG_CYTHON_DISABLE'] = self.__class__.__name__
         from borg.archiver import Archiver
         parser = Archiver().build_parser(prog='borg')
-        # return to regular Cython configuration, if we changed it
-        if os.environ.get('BORG_CYTHON_DISABLE') == self.__class__.__name__:
-            del os.environ['BORG_CYTHON_DISABLE']
         choices = {}
         for action in parser._actions:
             if action.choices is not None:
@@ -166,6 +163,9 @@ class build_usage(Command):
                 doc.write(re.sub("^", "    ", parser.format_help(), flags=re.M))
                 doc.write("\nDescription\n~~~~~~~~~~~\n")
                 doc.write(epilog)
+        # return to regular Cython configuration, if we changed it
+        if os.environ.get('BORG_CYTHON_DISABLE') == self.__class__.__name__:
+            del os.environ['BORG_CYTHON_DISABLE']
 
 
 class build_api(Command):

From d68451574bb3bfcf8cc339dc64386626464355c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 17:14:25 -0400
Subject: [PATCH 68/70] more debug

---
 setup.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/setup.py b/setup.py
index f2a3f8bb9..99fab07d8 100644
--- a/setup.py
+++ b/setup.py
@@ -140,6 +140,9 @@ class build_usage(Command):
         # allows us to build docs without the C modules fully loaded during help generation
         if 'BORG_CYTHON_DISABLE' not in os.environ:
             os.environ['BORG_CYTHON_DISABLE'] = self.__class__.__name__
+        from borg.helpers import have_cython
+        print('have_cython? %s' % have_cython())
+        print("CYTHON ENV: %s" % os.environ.get('BORG_CYTHON_DISABLE'))
         from borg.archiver import Archiver
         parser = Archiver().build_parser(prog='borg')
         choices = {}

From 80e53fb66de0e647f9b06e9f3f2d5bf8bf5d3395 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 17:16:11 -0400
Subject: [PATCH 69/70] it's a function, call it as such

---
 borg/remote.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/borg/remote.py b/borg/remote.py
index 466d627da..5d8c71a88 100644
--- a/borg/remote.py
+++ b/borg/remote.py
@@ -13,7 +13,7 @@ from . import __version__
 from .helpers import Error, IntegrityError, have_cython
 from .repository import Repository
 
-if have_cython:
+if have_cython():
     import msgpack
 
 BUFSIZE = 10 * 1024 * 1024

From 1c61f87da3a1128afcf4c03b157477de3948dfe5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= <anarcat@koumbit.org>
Date: Thu, 8 Oct 2015 17:20:52 -0400
Subject: [PATCH 70/70] remove debugging code and fix all have_cython calls

---
 borg/cache.py | 2 +-
 borg/fuse.py  | 2 +-
 setup.py      | 3 ---
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/borg/cache.py b/borg/cache.py
index 0f559f15e..4df4b7721 100644
--- a/borg/cache.py
+++ b/borg/cache.py
@@ -15,7 +15,7 @@ from .helpers import Error, get_cache_dir, decode_dict, st_mtime_ns, unhexlify,
 from .locking import UpgradableLock
 from .hashindex import ChunkIndex
 
-if have_cython:
+if have_cython():
     import msgpack
 
 
diff --git a/borg/fuse.py b/borg/fuse.py
index d7a5aa5ce..6f98f0a01 100644
--- a/borg/fuse.py
+++ b/borg/fuse.py
@@ -10,7 +10,7 @@ from .archive import Archive
 from .helpers import daemonize, have_cython
 from .remote import cache_if_remote
 
-if have_cython:
+if have_cython():
     import msgpack
 
 # Does this version of llfuse support ns precision?
diff --git a/setup.py b/setup.py
index 99fab07d8..f2a3f8bb9 100644
--- a/setup.py
+++ b/setup.py
@@ -140,9 +140,6 @@ class build_usage(Command):
         # allows us to build docs without the C modules fully loaded during help generation
         if 'BORG_CYTHON_DISABLE' not in os.environ:
             os.environ['BORG_CYTHON_DISABLE'] = self.__class__.__name__
-        from borg.helpers import have_cython
-        print('have_cython? %s' % have_cython())
-        print("CYTHON ENV: %s" % os.environ.get('BORG_CYTHON_DISABLE'))
         from borg.archiver import Archiver
         parser = Archiver().build_parser(prog='borg')
         choices = {}