From 440edf0227711def007372f1b45181a6249ec309 Mon Sep 17 00:00:00 2001 From: Mario Fetka Date: Fri, 11 Sep 2020 13:42:22 +0200 Subject: [PATCH] Imported Upstream version 11.5 --- .version | 2 +- HISTORY | 18 ++++++- Makefile.am | 5 +- Makefile.in | 5 +- README | 5 +- TODO | 12 ++--- acinclude.m4 | 4 +- cmdline/check.c | 96 ++++++++++++++++++++++-------------- cmdline/device.c | 2 +- cmdline/elem.h | 8 +-- cmdline/io.c | 13 +++++ cmdline/io.h | 20 ++++---- cmdline/metro.c | 108 +++++++++++++++++++++++++++++++++++++++++ cmdline/mingw.c | 4 +- cmdline/parity.c | 48 +++++++++++++++++- cmdline/parity.h | 15 ++++++ cmdline/pool.c | 2 +- cmdline/portable.h | 16 +++--- cmdline/scan.c | 2 +- cmdline/scrub.c | 3 +- cmdline/selftest.c | 2 +- cmdline/snapraid.c | 4 ++ cmdline/speed.c | 9 ++++ cmdline/state.c | 64 +++++++++++++++++------- cmdline/state.h | 2 +- cmdline/stream.c | 8 +-- cmdline/stream.h | 2 +- cmdline/support.c | 15 +++--- cmdline/support.h | 4 +- cmdline/sync.c | 4 +- cmdline/unix.c | 2 +- cmdline/unix.h | 2 +- cmdline/util.c | 35 ++++++++++++- cmdline/util.h | 3 +- config.h.in | 14 +++--- configure | 88 +++++++++++---------------------- configure.ac | 7 +-- raid/check.c | 2 +- raid/helper.h | 2 +- raid/internal.h | 4 +- raid/memory.h | 2 +- raid/raid.c | 16 +++--- raid/tables.c | 4 +- raid/test/Makefile | 2 +- snapraid.1 | 11 +++-- snapraid.d | 9 ++-- snapraid.txt | 9 ++-- tommyds/tommychain.h | 4 +- tommyds/tommyhash.c | 2 +- tommyds/tommyhash.h | 8 +-- tommyds/tommyhashdyn.h | 2 +- tommyds/tommylist.h | 4 +- tommyds/tommytree.h | 4 +- tommyds/tommytypes.h | 2 +- 54 files changed, 506 insertions(+), 234 deletions(-) create mode 100644 cmdline/metro.c diff --git a/.version b/.version index f2f3fcc..48cf2f2 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -11.3 \ No newline at end of file +11.5 \ No newline at end of file diff --git a/HISTORY b/HISTORY index ad7411f..65a6d9c 100644 --- a/HISTORY +++ b/HISTORY @@ -1,6 +1,20 @@ SnapRAID HISTORY ================ +11.5 2020/05 +============ + * Removed the default -march=native to allow to deploy in any machine. + * Fixed typos [Andrea Gelmini] + +11.4 2020/05 +============ + * Fix build errors due new gcc 10 default for -fno-common. + * In fixing, if a parity is filtered out, don't attempt to recover its size, + and proceed without it if missing. + * Avoid unnecessary parity read when fixing the parity itself. + This improves the 'fix' speed when a parity file is completely missing. + * Removed a build warning about major/minor defined now in sys/sysmacros.h. + 11.3 2018/11 ============ * Fixed handing of Linux devices that have multiple slaves. This affects @@ -139,7 +153,7 @@ SnapRAID HISTORY A workaround was to just run 'sync' one time with the -N, --force-nocopy option to disable the copy detection. * Restored the -O2 optimization option for Windows binaries, as -Og has - a too big performance penality. + a too big performance penalty. 9.2 2016/01 =========== @@ -412,7 +426,7 @@ SnapRAID HISTORY and Opteron. * Faster RAID5 and RAID6 implementation for ARM 64 bit CPUs. * If a silent error is found during a "sync" command, directly marks - the block as bad like in "scrub", without stopping the the "sync" + the block as bad like in "scrub", without stopping the "sync" process. * Sort files by inode when listing the directory. This improves the scanning performance. diff --git a/Makefile.am b/Makefile.am index 4c0c167..447d96b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -83,6 +83,7 @@ noinst_HEADERS = \ cmdline/murmur3test.c \ cmdline/spooky2.c \ cmdline/spooky2test.c \ + cmdline/metro.c \ cmdline/fnmatch.h \ cmdline/import.h \ cmdline/search.h \ @@ -158,7 +159,7 @@ maintainer-clean-local: CHECKFLAGS_COMMON = --test-skip-device --test-skip-self --test-force-progress --no-warnings --test-parity-limit=3333333 # --test-force-order-alpha -# Ensures to process files always in the same order despites +# Ensures to process files always in the same order despite # the inode, physical location, and dir order assigned by the OS. CHECKFLAGS_ALPHA = $(CHECKFLAGS_COMMON) --test-force-order-alpha @@ -499,7 +500,7 @@ endif $(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-run "rm bench/disk1/RUN-RM" --test-expect-failure sync echo RUN > bench/disk1/RUN-CHMOD if HAVE_POSIX -# Doesn't run this test as root because the root user overrride permissions +# Doesn't run this test as root because the root user override permissions if [[ $$EUID -ne 0 ]]; then \ $(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-run "chmod a-r bench/disk1/RUN-CHMOD" --test-expect-failure sync; \ fi diff --git a/Makefile.in b/Makefile.in index 73f8ac5..9524fd5 100644 --- a/Makefile.in +++ b/Makefile.in @@ -429,6 +429,7 @@ noinst_HEADERS = \ cmdline/murmur3test.c \ cmdline/spooky2.c \ cmdline/spooky2test.c \ + cmdline/metro.c \ cmdline/fnmatch.h \ cmdline/import.h \ cmdline/search.h \ @@ -487,7 +488,7 @@ man_MANS = snapraid.1 CHECKFLAGS_COMMON = --test-skip-device --test-skip-self --test-force-progress --no-warnings --test-parity-limit=3333333 # --test-force-order-alpha -# Ensures to process files always in the same order despites +# Ensures to process files always in the same order despite # the inode, physical location, and dir order assigned by the OS. CHECKFLAGS_ALPHA = $(CHECKFLAGS_COMMON) --test-force-order-alpha @@ -1409,7 +1410,7 @@ check-local: @HAVE_THREAD_CHECKER_FALSE@ echo RUN > bench/disk1/RUN-RM @HAVE_THREAD_CHECKER_FALSE@ $(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-run "rm bench/disk1/RUN-RM" --test-expect-failure sync @HAVE_THREAD_CHECKER_FALSE@ echo RUN > bench/disk1/RUN-CHMOD -# Doesn't run this test as root because the root user overrride permissions +# Doesn't run this test as root because the root user override permissions @HAVE_POSIX_TRUE@@HAVE_THREAD_CHECKER_FALSE@ if [[ $$EUID -ne 0 ]]; then \ @HAVE_POSIX_TRUE@@HAVE_THREAD_CHECKER_FALSE@ $(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-run "chmod a-r bench/disk1/RUN-CHMOD" --test-expect-failure sync; \ @HAVE_POSIX_TRUE@@HAVE_THREAD_CHECKER_FALSE@ fi diff --git a/README b/README index 52a90b7..9033002 100644 --- a/README +++ b/README @@ -11,6 +11,8 @@ have a lot of big files that rarely change. Beside the ability to recover from disk failures, the other features of SnapRAID are: +* You can use disk already filled with files, without the need to + reformat them. You will access them like now. * All your data is hashed to ensure data integrity and to avoid silent corruption. * If the failed disks are too many to allow a recovery, @@ -18,12 +20,11 @@ features of SnapRAID are: All the data in the other disks is safe. * If you accidentally delete some files in a disk, you can recover them. -* You can start with already filled disks. * The disks can have different sizes. * You can add disks at any time. * It doesn't lock-in your data. You can stop using SnapRAID at any time without the need to reformat or move data. -* To access a file, a single disk needs to spin, saving power and +* To access a file, only a single disk needs to spin, saving power and producing less noise. The official site of SnapRAID is: diff --git a/TODO b/TODO index f02c44c..329b417 100644 --- a/TODO +++ b/TODO @@ -25,8 +25,8 @@ with a mono thread implementation with 100.0000 files. * Support more parity levels It can be done with a generic computation function, using intrinsic for SSSE3 and AVX instructions. -It would be intersting to compare performance with the hand-written -assembler functions. Eventially we can convert them to use intrinsic also. +It would be interesting to compare performance with the hand-written +assembler functions. Eventually we can convert them to use intrinsic also. https://sourceforge.net/p/snapraid/discussion/1677233/thread/9dbd7581/ * Extend haspdeep to support the SnapRAID hash : @@ -151,7 +151,7 @@ See: https://sourceforge.net/p/snapraid/discussion/1677233/thread/cdea773f/ * Allocate parity minimizing concurrent use of it Each parity allocation should check for a parity range with less utilization by other disks. -We need to take care do disable this meachnism when the parity space +We need to take care do disable this mechanism when the parity space is near to fillup the parity partition. See: https://sourceforge.net/p/snapraid/discussion/1677233/thread/1797bf7d/ + This increase the possibility of recovering with multiple failures with not @@ -182,7 +182,7 @@ allowing the user to choose where to put it. - It won't work with disks of different size. Suppose to have all disks of size N, with only one of size M>N. To fully use the M space, you can allocate a full N parity in such disk, -but the remaning space will also need additional parity in the other disks, +but the remaining space will also need additional parity in the other disks, in fact requiring a total of M parity for the array. In the end, we cannot avoid that the first biggest disk added is fully dedicated to parity, even if it means to leave some space unused. @@ -199,7 +199,7 @@ But it should be only few bits for each file. So, it should be manageable. A lot of discussions about this feature :) https://sourceforge.net/p/snapraid/discussion/1677233/thread/b2cd9385/ - The only benefit is to distribute better the data. This could help the recovery process, -in case of multiple failures. But no real usability or funtionality benefit in the normal +in case of multiple failures. But no real usability or functionality benefit in the normal case. * https://sourceforge.net/p/snapraid/discussion/1677233/thread/2cb97e8a/ @@ -229,7 +229,7 @@ are automatically deleted. * Checks if splitting hash/parity computation in 4K pages can improve speed in sync. That should increase cache locality, -because we read the data two times for hash and and parity, +because we read the data two times for hash and parity, and if we manage to keep it in the cache, we should save time. - We now hash first the faster disks, and this could reduce performance as we'll have to wait for all disks. diff --git a/acinclude.m4 b/acinclude.m4 index 1b92cc9..96e9c5f 100644 --- a/acinclude.m4 +++ b/acinclude.m4 @@ -1,7 +1,7 @@ dnl @synopsis AC_CHECK_CC_OPT(flag, ifyes, ifno) dnl -dnl Shows a message as like "checking wether gcc accepts flag ... no" -dnl and executess ifyes or ifno. +dnl Shows a message as like "checking whether gcc accepts flag ... no" +dnl and executes ifyes or ifno. AC_DEFUN([AC_CHECK_CC_OPT], [ diff --git a/cmdline/check.c b/cmdline/check.c index 72012df..b3d2048 100644 --- a/cmdline/check.c +++ b/cmdline/check.c @@ -574,7 +574,7 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign } else { log_tag("recover_unsync:%u:%u: Skipped for%s%s\n", pos, n, !something_to_recover ? " nothing to recover" : "", - !something_unsynced ? " nothing unsynched" : "" + !something_unsynced ? " nothing unsynced" : "" ); } @@ -1441,7 +1441,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna char path[PATH_MAX]; struct stat st; struct snapraid_file* file; - int unsuccesful = 0; + int unsuccessful = 0; file = node->data; node = node->next; /* next node */ @@ -1460,24 +1460,24 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna pathprint(path, sizeof(path), "%s%s", disk->dir, file->sub); ret = stat(path, &st); if (ret == -1) { - unsuccesful = 1; + unsuccessful = 1; log_error("Error stating empty file '%s'. %s.\n", path, strerror(errno)); log_tag("error:%s:%s: Empty file stat error\n", disk->name, esc_tag(file->sub, esc_buffer)); ++error; } else if (!S_ISREG(st.st_mode)) { - unsuccesful = 1; + unsuccessful = 1; log_tag("error:%s:%s: Empty file error for not regular file\n", disk->name, esc_tag(file->sub, esc_buffer)); ++error; } else if (st.st_size != 0) { - unsuccesful = 1; + unsuccessful = 1; log_tag("error:%s:%s: Empty file error for size '%" PRIu64 "'\n", disk->name, esc_tag(file->sub, esc_buffer), (uint64_t)st.st_size); ++error; } - if (fix && unsuccesful) { + if (fix && unsuccessful) { int f; /* create the ancestor directories */ @@ -1552,7 +1552,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna struct stat st; struct stat stto; struct snapraid_link* slink; - int unsuccesful = 0; + int unsuccessful = 0; int unrecoverable = 0; slink = node->data; @@ -1568,13 +1568,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna pathprint(path, sizeof(path), "%s%s", disk->dir, slink->sub); ret = stat(path, &st); if (ret == -1) { - unsuccesful = 1; + unsuccessful = 1; log_error("Error stating hardlink '%s'. %s.\n", path, strerror(errno)); log_tag("hardlink_error:%s:%s:%s: Hardlink stat error\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt)); ++error; } else if (!S_ISREG(st.st_mode)) { - unsuccesful = 1; + unsuccessful = 1; log_tag("hardlink_error:%s:%s:%s: Hardlink error for not regular file\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt)); ++error; @@ -1584,7 +1584,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna pathprint(pathto, sizeof(pathto), "%s%s", disk->dir, slink->linkto); ret = stat(pathto, &stto); if (ret == -1) { - unsuccesful = 1; + unsuccessful = 1; if (errno == ENOENT) { unrecoverable = 1; @@ -1604,12 +1604,12 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna log_tag("hardlink_error:%s:%s:%s: Hardlink to stat error\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt)); ++error; } else if (!S_ISREG(stto.st_mode)) { - unsuccesful = 1; + unsuccessful = 1; log_tag("hardlink_error:%s:%s:%s: Hardlink-to error for not regular file\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt)); ++error; - } else if (!unsuccesful && st.st_ino != stto.st_ino) { - unsuccesful = 1; + } else if (!unsuccessful && st.st_ino != stto.st_ino) { + unsuccessful = 1; log_error("Mismatch hardlink '%s' and '%s'. Different inode.\n", path, pathto); log_tag("hardlink_error:%s:%s:%s: Hardlink mismatch for different inode\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt)); @@ -1620,13 +1620,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna pathprint(path, sizeof(path), "%s%s", disk->dir, slink->sub); ret = readlink(path, linkto, sizeof(linkto)); if (ret < 0) { - unsuccesful = 1; + unsuccessful = 1; log_error("Error reading symlink '%s'. %s.\n", path, strerror(errno)); log_tag("symlink_error:%s:%s: Symlink read error\n", disk->name, esc_tag(slink->sub, esc_buffer)); ++error; } else if (ret >= PATH_MAX) { - unsuccesful = 1; + unsuccessful = 1; log_error("Error reading symlink '%s'. Symlink too long.\n", path); log_tag("symlink_error:%s:%s: Symlink read error\n", disk->name, esc_tag(slink->sub, esc_buffer)); @@ -1635,7 +1635,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna linkto[ret] = 0; if (strcmp(linkto, slink->linkto) != 0) { - unsuccesful = 1; + unsuccessful = 1; log_tag("symlink_error:%s:%s: Symlink data error '%s' instead of '%s'\n", disk->name, esc_tag(slink->sub, esc_buffer), linkto, slink->linkto); ++error; @@ -1643,7 +1643,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna } } - if (fix && unsuccesful && !unrecoverable) { + if (fix && unsuccessful && !unrecoverable) { /* create the ancestor directories */ ret = mkancestor(path); if (ret != 0) { @@ -1720,7 +1720,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna char path[PATH_MAX]; struct stat st; struct snapraid_dir* dir; - int unsuccesful = 0; + int unsuccessful = 0; dir = node->data; node = node->next; /* next node */ @@ -1734,19 +1734,19 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna pathprint(path, sizeof(path), "%s%s", disk->dir, dir->sub); ret = stat(path, &st); if (ret == -1) { - unsuccesful = 1; + unsuccessful = 1; log_error("Error stating dir '%s'. %s.\n", path, strerror(errno)); log_tag("dir_error:%s:%s: Dir stat error\n", disk->name, esc_tag(dir->sub, esc_buffer)); ++error; } else if (!S_ISDIR(st.st_mode)) { - unsuccesful = 1; + unsuccessful = 1; log_tag("dir_error:%s:%s: Dir error for not directory\n", disk->name, esc_tag(dir->sub, esc_buffer)); ++error; } - if (fix && unsuccesful) { + if (fix && unsuccessful) { /* create the ancestor directories */ ret = mkancestor(path); if (ret != 0) { @@ -1973,20 +1973,32 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b } parity_ptr[l] = &parity[l]; - ret = parity_create(parity_ptr[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size); - if (ret == -1) { - /* LCOV_EXCL_START */ - log_fatal("WARNING! Without an accessible %s file, it isn't possible to fix any error.\n", lev_name(l)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - ret = parity_chsize(parity_ptr[l], &state->parity[l], 0, size, state->block_size, state->opt.skip_fallocate, state->opt.skip_space_holder); - if (ret == -1) { - /* LCOV_EXCL_START */ - log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ + /* if the parity is excluded */ + if (state->parity[l].is_excluded_by_filter) { + /* open for reading, and ignore error */ + ret = parity_open(parity_ptr[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size); + if (ret == -1) { + /* continue anyway */ + parity_ptr[l] = 0; + } + } else { + /* open for writing */ + ret = parity_create(parity_ptr[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size); + if (ret == -1) { + /* LCOV_EXCL_START */ + log_fatal("WARNING! Without an accessible %s file, it isn't possible to fix any error.\n", lev_name(l)); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + ret = parity_chsize(parity_ptr[l], &state->parity[l], 0, size, state->block_size, state->opt.skip_fallocate, state->opt.skip_space_holder); + if (ret == -1) { + /* LCOV_EXCL_START */ + log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l)); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } } } } else if (!state->opt.auditonly) { @@ -2030,14 +2042,26 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b /* try to close only if opened */ for (l = 0; l < state->level; ++l) { if (parity_ptr[l]) { + /* if fixing and not excluded, truncate parity not valid */ + if (fix && !state->parity[l].is_excluded_by_filter) { + ret = parity_truncate(parity_ptr[l]); + if (ret == -1) { + /* LCOV_EXCL_START */ + log_fatal("DANGER! Unexpected truncate error in %s disk.\n", lev_name(l)); + ++error; + /* continue, as we are already exiting */ + /* LCOV_EXCL_STOP */ + } + } + ret = parity_close(parity_ptr[l]); - /* LCOV_EXCL_START */ if (ret == -1) { + /* LCOV_EXCL_START */ log_fatal("DANGER! Unexpected close error in %s disk.\n", lev_name(l)); ++error; /* continue, as we are already exiting */ + /* LCOV_EXCL_STOP */ } - /* LCOV_EXCL_STOP */ } } diff --git a/cmdline/device.c b/cmdline/device.c index f170ad6..605fdc3 100644 --- a/cmdline/device.c +++ b/cmdline/device.c @@ -818,7 +818,7 @@ static void state_smart(unsigned n, tommy_list* low) /* |<##################################################################72>|####80>| */ printf("These values are the probabilities that in the next year you'll have a\n"); - printf("sequence of failures that the parity WONT be able to recover, assuming\n"); + printf("sequence of failures that the parity WON'T be able to recover, assuming\n"); printf("that you regularly scrub, and in case repair, the array in the specified\n"); printf("time.\n"); diff --git a/cmdline/elem.h b/cmdline/elem.h index e821e41..0c4a555 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -349,7 +349,7 @@ struct snapraid_disk { block_off_t first_free_block; int has_volatile_inodes; /**< If the underline file-system has not persistent inodes. */ - int has_volatile_hardlinks; /**< If the underline file-system has not syncronized metadata for hardlink (NTFS). */ + int has_volatile_hardlinks; /**< If the underline file-system has not synchronized metadata for hardlink (NTFS). */ int has_unreliable_physical; /**< If the physical offset of files has duplicates. */ int has_different_uuid; /**< If the disk has a different UUID, meaning that it is not the same file-system. */ int has_unsupported_uuid; /**< If the disk doesn't report UUID, meaning it's not supported. */ @@ -1000,7 +1000,7 @@ int fs_check(struct snapraid_disk* disk); * After this call you can use the par2file/par2block operations * to query the relation. * - * \note This function is NOT thread-safe as it uses the the disk cache. + + * \note This function is NOT thread-safe as it uses the disk cache. + */ void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos); @@ -1010,7 +1010,7 @@ void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snap * After this call the par2file/par2block operations * won't find anymore the parity association. * - * \note This function is NOT thread-safe as it uses the the disk cache. + * \note This function is NOT thread-safe as it uses the disk cache. */ void fs_deallocate(struct snapraid_disk* disk, block_off_t pos); @@ -1129,7 +1129,7 @@ static inline snapraid_info info_make(time_t last_access, int error, int rehash, /** * Extract the time information. * This is the last time when the block was know to be correct. - * The "scrubbed" info tells if the time is referreing at the latest sync or scrub. + * The "scrubbed" info tells if the time is referring at the latest sync or scrub. */ static inline time_t info_get_time(snapraid_info info) { diff --git a/cmdline/io.c b/cmdline/io.c index c71ba41..156d512 100644 --- a/cmdline/io.c +++ b/cmdline/io.c @@ -19,6 +19,19 @@ #include "io.h" +void (*io_start)(struct snapraid_io* io, + block_off_t blockstart, block_off_t blockmax, + int (*block_is_enabled)(void* arg, block_off_t), void* blockarg) = 0; +void (*io_stop)(struct snapraid_io* io) = 0; +block_off_t (*io_read_next)(struct snapraid_io* io, void*** buffer) = 0; +struct snapraid_task* (*io_data_read)(struct snapraid_io* io, unsigned* diskcur, unsigned* waiting_map, unsigned* waiting_mac) = 0; +struct snapraid_task* (*io_parity_read)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac) = 0; +void (*io_parity_write)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac) = 0; +void (*io_write_preset)(struct snapraid_io* io, block_off_t blockcur, int skip) = 0; +void (*io_write_next)(struct snapraid_io* io, block_off_t blockcur, int skip, int* writer_error) = 0; +void (*io_refresh)(struct snapraid_io* io) = 0; + + /** * Get the next block position to process. */ diff --git a/cmdline/io.h b/cmdline/io.h index 82ec744..a0cfb63 100644 --- a/cmdline/io.h +++ b/cmdline/io.h @@ -306,21 +306,21 @@ void io_init(struct snapraid_io* io, struct snapraid_state* state, struct snapraid_parity_handle* parity_handle_map, unsigned parity_handle_max); /** - * Deinitialize te InputOutput workers. + * Deinitialize the InputOutput workers. */ void io_done(struct snapraid_io* io); /** * Start all the worker threads. */ -void (*io_start)(struct snapraid_io* io, +extern void (*io_start)(struct snapraid_io* io, block_off_t blockstart, block_off_t blockmax, int (*block_is_enabled)(void* arg, block_off_t), void* blockarg); /** * Stop all the worker threads. */ -void (*io_stop)(struct snapraid_io* io); +extern void (*io_stop)(struct snapraid_io* io); /** * Next read position. @@ -332,7 +332,7 @@ void (*io_stop)(struct snapraid_io* io); * \param buffer The data buffers to use for this position. * \return The parity position. */ -block_off_t (*io_read_next)(struct snapraid_io* io, void*** buffer); +extern block_off_t (*io_read_next)(struct snapraid_io* io, void*** buffer); /** * Read a data block. @@ -343,7 +343,7 @@ block_off_t (*io_read_next)(struct snapraid_io* io, void*** buffer); * \param diskcur The position of the data block in the ::handle_map vector. * \return The completed task. */ -struct snapraid_task* (*io_data_read)(struct snapraid_io* io, unsigned* diskcur, unsigned* waiting_map, unsigned* waiting_mac); +extern struct snapraid_task* (*io_data_read)(struct snapraid_io* io, unsigned* diskcur, unsigned* waiting_map, unsigned* waiting_mac); /** * Read a parity block. @@ -354,7 +354,7 @@ struct snapraid_task* (*io_data_read)(struct snapraid_io* io, unsigned* diskcur, * \param levcur The position of the parity block in the ::parity_handle_map vector. * \return The completed task. */ -struct snapraid_task* (*io_parity_read)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac); +extern struct snapraid_task* (*io_parity_read)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac); /** * Write of a parity block. @@ -364,7 +364,7 @@ struct snapraid_task* (*io_parity_read)(struct snapraid_io* io, unsigned* levcur * \param io InputOutput context. * \param levcur The position of the parity block in the ::parity_handle_map vector. */ -void (*io_parity_write)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac); +extern void (*io_parity_write)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac); /** * Preset the write position. @@ -376,7 +376,7 @@ void (*io_parity_write)(struct snapraid_io* io, unsigned* levcur, unsigned* wait * \param blockcur The parity position to write. * \param skip Skip the writes, in case parity doesn't need to be updated. */ -void (*io_write_preset)(struct snapraid_io* io, block_off_t blockcur, int skip); +extern void (*io_write_preset)(struct snapraid_io* io, block_off_t blockcur, int skip); /** * Next write position. @@ -389,12 +389,12 @@ void (*io_write_preset)(struct snapraid_io* io, block_off_t blockcur, int skip); * \param skip Skip the writes, in case parity doesn't need to be updated. * \param writer_error Return the number of errors. Vector of IO_WRITER_ERROR_MAX elements. */ -void (*io_write_next)(struct snapraid_io* io, block_off_t blockcur, int skip, int* writer_error); +extern void (*io_write_next)(struct snapraid_io* io, block_off_t blockcur, int skip, int* writer_error); /** * Refresh the number of cached blocks for all data and parity disks. */ -void (*io_refresh)(struct snapraid_io* io); +extern void (*io_refresh)(struct snapraid_io* io); #endif diff --git a/cmdline/metro.c b/cmdline/metro.c new file mode 100644 index 0000000..18f4027 --- /dev/null +++ b/cmdline/metro.c @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2019 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * Derivative work from metrohash128.cpp + * + * metrohash128.cpp + * + * Copyright 2015-2018 J. Andrew Rogers + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +static const uint64_t k0 = 0xC83A91E1; +static const uint64_t k1 = 0x8648DBDB; +static const uint64_t k2 = 0x7BDEC03B; +static const uint64_t k3 = 0x2F5870A5; + +void MetroHash128(const void* data, size_t size, const uint8_t* seed, uint8_t* digest) +{ + const uint8_t* ptr = data; + uint64_t v[4]; + + v[0] = (util_read64(seed) - k0) * k3; + v[1] = (util_read64(seed + 8) + k1) * k2; + + if (size >= 32) { + v[2] = (util_read64(seed) + k0) * k2; + v[3] = (util_read64(seed + 8) - k1) * k3; + + do { + v[0] += util_read64(ptr) * k0; ptr += 8; v[0] = util_rotr64(v[0], 29) + v[2]; + v[1] += util_read64(ptr) * k1; ptr += 8; v[1] = util_rotr64(v[1], 29) + v[3]; + v[2] += util_read64(ptr) * k2; ptr += 8; v[2] = util_rotr64(v[2], 29) + v[0]; + v[3] += util_read64(ptr) * k3; ptr += 8; v[3] = util_rotr64(v[3], 29) + v[1]; + size -= 32; + } while (size >= 32); + + v[2] ^= util_rotr64(((v[0] + v[3]) * k0) + v[1], 21) * k1; + v[3] ^= util_rotr64(((v[1] + v[2]) * k1) + v[0], 21) * k0; + v[0] ^= util_rotr64(((v[0] + v[2]) * k0) + v[3], 21) * k1; + v[1] ^= util_rotr64(((v[1] + v[3]) * k1) + v[2], 21) * k0; + } + + if (size >= 16) { + v[0] += util_read64(ptr) * k2; ptr += 8; v[0] = util_rotr64(v[0], 33) * k3; + v[1] += util_read64(ptr) * k2; ptr += 8; v[1] = util_rotr64(v[1], 33) * k3; + v[0] ^= util_rotr64((v[0] * k2) + v[1], 45) * k1; + v[1] ^= util_rotr64((v[1] * k3) + v[0], 45) * k0; + size -= 16; + } + + if (size >= 8) { + v[0] += util_read64(ptr) * k2; ptr += 8; v[0] = util_rotr64(v[0], 33) * k3; + v[0] ^= util_rotr64((v[0] * k2) + v[1], 27) * k1; + size -= 8; + } + + if (size >= 4) { + v[1] += util_read32(ptr) * k2; ptr += 4; v[1] = util_rotr64(v[1], 33) * k3; + v[1] ^= util_rotr64((v[1] * k3) + v[0], 46) * k0; + size -= 4; + } + + if (size >= 2) { + v[0] += util_read16(ptr) * k2; ptr += 2; v[0] = util_rotr64(v[0], 33) * k3; + v[0] ^= util_rotr64((v[0] * k2) + v[1], 22) * k1; + size -= 2; + } + + if (size >= 1) { + v[1] += util_read8(ptr) * k2; v[1] = util_rotr64(v[1], 33) * k3; + v[1] ^= util_rotr64((v[1] * k3) + v[0], 58) * k0; + } + + v[0] += util_rotr64((v[0] * k0) + v[1], 13); + v[1] += util_rotr64((v[1] * k1) + v[0], 37); + v[0] += util_rotr64((v[0] * k2) + v[1], 13); + v[1] += util_rotr64((v[1] * k3) + v[0], 37); + + util_write64(digest, v[0]); + util_write64(digest + 8, v[0]); +} + diff --git a/cmdline/mingw.c b/cmdline/mingw.c index ef10ee8..5b8c9d4 100644 --- a/cmdline/mingw.c +++ b/cmdline/mingw.c @@ -1556,7 +1556,7 @@ int windows_link(const char* existing, const char* file) } /** - * In Windows 10 allow creationg of symblink by not priviliged user. + * In Windows 10 allow creation of symblink by not privileged user. * * See: Symlinks in Windows 10! * https://blogs.windows.com/buildingapps/2016/12/02/symlinks-windows-10/#cQG7cx48oGH86lkI.97 @@ -1724,7 +1724,7 @@ int fsinfo(const char* path, int* has_persistent_inode, int* has_syncronized_har if (has_persistent_inode) *has_persistent_inode = 1; - /* NTFS doesn't syncronize hardlinks metadata */ + /* NTFS doesn't synchronize hardlinks metadata */ if (has_syncronized_hardlinks) *has_syncronized_hardlinks = 0; diff --git a/cmdline/parity.c b/cmdline/parity.c index 6bfdbcd..e4569fe 100644 --- a/cmdline/parity.c +++ b/cmdline/parity.c @@ -217,6 +217,9 @@ int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_p /* LCOV_EXCL_STOP */ } + /* the initial valid size is the size on disk */ + split->valid_size = split->st.st_size; + /** * If the parity size is not yet set, set it now. * This happens when expanding the number of parities, @@ -528,6 +531,10 @@ static int parity_handle_chsize(struct snapraid_split_handle* split, data_off_t /* LCOV_EXCL_STOP */ } + /* if we shrink, update the valid size, but don't update when growing */ + if (split->valid_size > split->st.st_size) + split->valid_size = split->st.st_size; + return 0; } @@ -697,6 +704,9 @@ int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_par /* LCOV_EXCL_STOP */ } + /* the initial valid size is the size on disk */ + split->valid_size = split->st.st_size; + /** * If the parity size is not yet set, set it now. * This happens when expanding the number of parities, @@ -751,7 +761,7 @@ int parity_sync(struct snapraid_parity_handle* handle) ret = fsync(split->f); if (ret != 0) { /* LCOV_EXCL_START */ - log_fatal("Error synching parity file '%s'. %s.\n", split->path, strerror(errno)); + log_fatal("Error syncing parity file '%s'. %s.\n", split->path, strerror(errno)); return -1; /* LCOV_EXCL_STOP */ } @@ -761,6 +771,30 @@ int parity_sync(struct snapraid_parity_handle* handle) return 0; } +int parity_truncate(struct snapraid_parity_handle* handle) +{ + unsigned s; + int f_ret = 0; + + for (s = 0; s < handle->split_mac; ++s) { + struct snapraid_split_handle* split = &handle->split_map[s]; + int ret; + + /* truncate any data that we know it's not valid */ + ret = ftruncate(split->f, split->valid_size); + if (ret != 0) { + /* LCOV_EXCL_START */ + log_fatal("Error truncating the parity file '%s' to size %" PRIu64 ". %s.\n", split->path, split->valid_size, strerror(errno)); + f_ret = -1; + /* LCOV_EXCL_STOP */ + + /* continue to truncate the others */ + } + } + + return f_ret; +} + int parity_close(struct snapraid_parity_handle* handle) { unsigned s; @@ -826,6 +860,10 @@ int parity_write(struct snapraid_parity_handle* handle, block_off_t pos, unsigne /* LCOV_EXCL_STOP */ } + /* update the valid range */ + if (split->valid_size < offset + block_size) + split->valid_size = offset + block_size; + write_ret = pwrite(split->f, block_buffer, block_size, offset); if (write_ret != (ssize_t)block_size) { /* conversion is safe because block_size is always small */ /* LCOV_EXCL_START */ @@ -867,6 +905,14 @@ int parity_read(struct snapraid_parity_handle* handle, block_off_t pos, unsigned /* LCOV_EXCL_STOP */ } + /* if read is completely out of the valid range */ + if (offset >= split->valid_size) { + /* LCOV_EXCL_START */ + out("Missing data reading file '%s' at offset %" PRIu64 " for size %u.\n", split->path, offset, block_size); + return -1; + /* LCOV_EXCL_STOP */ + } + count = 0; do { read_ret = pread(split->f, block_buffer + count, block_size - count, offset + count); diff --git a/cmdline/parity.h b/cmdline/parity.h index 056362e..3380eb7 100644 --- a/cmdline/parity.h +++ b/cmdline/parity.h @@ -36,6 +36,16 @@ struct snapraid_split_handle { */ data_off_t size; + /** + * Valid size of the parity split. + * This is the size effectively written, and not the result of a chsize operation. + * It's used to make read operations failing if read over that size. + * + * Parity is also truncated to that size when fixing it, in case of a Break (Ctrl+C) + * of the program. + */ + data_off_t valid_size; + /** * Artificial size limit for testing. * 0 means unlimited. @@ -113,6 +123,11 @@ int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_par */ int parity_sync(struct snapraid_parity_handle* handle); +/** + * Truncate the parity file to the valid size. + */ +int parity_truncate(struct snapraid_parity_handle* handle); + /** * Close the parity file. */ diff --git a/cmdline/pool.c b/cmdline/pool.c index 44e3510..85cfc40 100644 --- a/cmdline/pool.c +++ b/cmdline/pool.c @@ -369,7 +369,7 @@ static void make_link(tommy_hashdyn* poolset, const char* pool_dir, const char* #ifdef _WIN32 } else if (errno == EPERM) { /* LCOV_EXCL_START */ - log_fatal("You must run as Adminstrator to be able to create symlinks.\n"); + log_fatal("You must run as Administrator to be able to create symlinks.\n"); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ #endif diff --git a/cmdline/portable.h b/cmdline/portable.h index 053ad85..5c75428 100644 --- a/cmdline/portable.h +++ b/cmdline/portable.h @@ -35,7 +35,7 @@ #define __USE_MINGW_ANSI_STDIO 1 /** - * Define the MSVCRT version targetting Windows Vista. + * Define the MSVCRT version targeting Windows Vista. */ #define __MSVCRT_VERSION__ 0x0600 @@ -198,9 +198,11 @@ #include #endif -#if MAJOR_IN_MKDEV +#if HAVE_SYS_MKDEV #include -#elif MAJOR_IN_SYSMACROS +#endif + +#if HAVE_SYS_SYSMACROS_H #include #endif @@ -387,14 +389,14 @@ void os_clear(void); * * If no log file is selected, it's 0. */ -FILE* stdlog; +extern FILE* stdlog; /** * Exit codes for testing. */ -int exit_success; -int exit_failure; -int exit_sync_needed; +extern int exit_success; +extern int exit_failure; +extern int exit_sync_needed; #undef EXIT_SUCCESS #undef EXIT_FAILURE #define EXIT_SUCCESS exit_success diff --git a/cmdline/scan.c b/cmdline/scan.c index 8e70ff3..f879cd3 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -501,7 +501,7 @@ static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struc * because the metadata in the directory is updated only when the file * is closed. * - * The same happens for hardlinks that duplicate metatada. + * The same happens for hardlinks that duplicate metadata. * The link metadata is updated only when the link is opened. * This extends also to st_size and st_nlink. * diff --git a/cmdline/scrub.c b/cmdline/scrub.c index 20a7ead..d69f3f0 100644 --- a/cmdline/scrub.c +++ b/cmdline/scrub.c @@ -665,7 +665,8 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par log_fatal("DANGER! Unexpected data errors! The failing blocks are now marked as bad!\n"); if (io_error || silent_error) { log_fatal("Use 'snapraid status' to list the bad blocks.\n"); - log_fatal("Use 'snapraid -e fix' to recover.\n"); + log_fatal("Use 'snapraid -e fix' to recover them.\n"); + log_fatal("Use 'snapraid -p bad scrub' to recheck after fixing.\n"); } log_tag("summary:error_file:%u\n", error); diff --git a/cmdline/selftest.c b/cmdline/selftest.c index b1e355a..8f79e5c 100644 --- a/cmdline/selftest.c +++ b/cmdline/selftest.c @@ -632,7 +632,7 @@ void selftest(void) } if (raid_test_par(RAID_MODE_CAUCHY, 1, 256) != 0) { /* LCOV_EXCL_START */ - log_fatal("Failed GEN Cauchy test sigle data disk\n"); + log_fatal("Failed GEN Cauchy test single data disk\n"); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index 07103a6..ec4f839 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -1001,6 +1001,10 @@ int main(int argc, char* argv[]) switch (operation) { case OPERATION_FIX : case OPERATION_CHECK : + case OPERATION_SMART : + case OPERATION_DEVICES : + case OPERATION_SPINUP : + case OPERATION_SPINDOWN : break; default : if (opt.force_device) { diff --git a/cmdline/speed.c b/cmdline/speed.c index f3ecdf8..73d2722 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -216,6 +216,7 @@ void speed(int period) printf("%8s", "best"); printf("%8s", "murmur3"); printf("%8s", "spooky2"); + printf("%8s", "metro"); printf("\n"); printf("%8s", "hash"); @@ -245,6 +246,14 @@ void speed(int period) memhash(HASH_SPOOKY2, seed, digest, v[j], size); } SPEED_STOP + printf("%8" PRIu64, ds / dt); + fflush(stdout); + + SPEED_START { + for (j = 0; j < nd; ++j) + memhash(HASH_METRO, seed, digest, v[j], size); + } SPEED_STOP + printf("%8" PRIu64, ds / dt); printf("\n"); printf("\n"); diff --git a/cmdline/state.c b/cmdline/state.c index b09fba2..3b04e96 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -279,7 +279,7 @@ static void state_config_check(struct snapraid_state* state, const char* path, t struct snapraid_disk* other = j->data; if (disk->device == other->device) { if (state->opt.force_device) { - /* note tha we just ignore the issue */ + /* note that we just ignore the issue */ /* and we DON'T mark the disk to be skipped */ /* because we want to use these disks */ if (!state->opt.no_warnings) @@ -310,15 +310,23 @@ static void state_config_check(struct snapraid_state* state, const char* path, t for (l = 0; l < state->level; ++l) { for (s = 0; s < state->parity[l].split_mac; ++s) { if (disk->device == state->parity[l].split_map[s].device) { - /* LCOV_EXCL_START */ - log_fatal("Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path); + if (state->opt.force_device) { + /* note that we just ignore the issue */ + /* and we DON'T mark the disk to be skipped */ + /* because we want to use these disks */ + if (!state->opt.no_warnings) + log_fatal("DANGER! Ignoring that disks '%s' and %s '%s' are on the same device\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path); + } else { + /* LCOV_EXCL_START */ + log_fatal("Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path); #ifdef _WIN32 - log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device); - log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n"); - log_fatal("to change one of the disk serial.\n"); + log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device); + log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n"); + log_fatal("to change one of the disk serial.\n"); #endif - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } } } } @@ -364,7 +372,7 @@ static void state_config_check(struct snapraid_state* state, const char* path, t for (t = 0; t < state->parity[j].split_mac; ++t) { if (state->parity[l].split_map[s].device == state->parity[j].split_map[t].device) { if (state->opt.force_device) { - /* note tha we just ignore the issue */ + /* note that we just ignore the issue */ /* and we DON'T mark the disk to be skipped */ /* because we want to use these disks */ if (!state->opt.no_warnings) @@ -623,7 +631,7 @@ void state_config(struct snapraid_state* state, const char* path, const char* co } state->block_size *= KIBI; } else if (strcmp(tag, "hashsize") == 0 - || strcmp(tag, "hash_size") == 0 /* v11.0 used incorretly this one, kept now for backward compatibility */ + || strcmp(tag, "hash_size") == 0 /* v11.0 used incorrectly this one, kept now for backward compatibility */ ) { uint32_t hash_size; @@ -1618,7 +1626,7 @@ static void state_content_check(struct snapraid_state* state, const char* path) /** * Check if the position is REQUIRED, or we can completely clear it from the state. * - * Note that position with only DELETED blocks are discarged. + * Note that position with only DELETED blocks are discharged. */ static int fs_position_is_required(struct snapraid_state* state, block_off_t pos) { @@ -1844,7 +1852,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (state->block_size == 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal("Internal incosistency due zero blocksize!\n"); + log_fatal("Internal inconsistency due zero blocksize!\n"); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -2398,6 +2406,9 @@ static void state_read_content(struct snapraid_state* state, const char* path, S case 'k' : state->hash = HASH_SPOOKY2; break; + case 'm' : + state->hash = HASH_METRO; + break; default : /* LCOV_EXCL_START */ decoding_error(path, f); @@ -2425,6 +2436,9 @@ static void state_read_content(struct snapraid_state* state, const char* path, S case 'k' : state->prevhash = HASH_SPOOKY2; break; + case 'm' : + state->prevhash = HASH_METRO; + break; default : /* LCOV_EXCL_START */ decoding_error(path, f); @@ -2947,6 +2961,8 @@ static void* state_write_thread(void* arg) sputc('u', f); } else if (state->hash == HASH_SPOOKY2) { sputc('k', f); + } else if (state->hash == HASH_METRO) { + sputc('m', f); } else { /* LCOV_EXCL_START */ log_fatal("Unexpected hash when writing the content file '%s'.\n", serrorfile(f)); @@ -2970,6 +2986,8 @@ static void* state_write_thread(void* arg) sputc('u', f); } else if (state->prevhash == HASH_SPOOKY2) { sputc('k', f); + } else if (state->prevhash == HASH_METRO) { + sputc('m', f); } else { /* LCOV_EXCL_START */ log_fatal("Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f)); @@ -3837,10 +3855,14 @@ struct state_verify_thread_context { static void* state_verify_thread(void* arg) { struct state_verify_thread_context* context = arg; + struct snapraid_content* content = context->content; STREAM* f = context->f; unsigned char buf[4]; uint32_t crc_stored; uint32_t crc_computed; + uint64_t start; + + start = tick_ms(); if (sdeplete(f, buf) != 0) { /* LCOV_EXCL_START */ @@ -3872,6 +3894,8 @@ static void* state_verify_thread(void* arg) /* LCOV_EXCL_STOP */ } + msg_progress("Verified %s in %" PRIu64 " seconds\n", content->content, (tick_ms() - start) / 1000); + return 0; } @@ -4155,7 +4179,7 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tom } } - /* if we are filtering by disk, exclude any parity not explicitely included */ + /* if we are filtering by disk, exclude any parity not explicitly included */ if (!tommy_list_empty(filterlist_disk)) { /* for each parity disk */ for (l = 0; l < state->level; ++l) { @@ -4463,7 +4487,8 @@ int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_o ) { time_t elapsed; unsigned out_perc = 0; - unsigned out_speed = 0; + unsigned out_size_speed = 0; + unsigned out_block_speed = 0; unsigned out_cpu = 0; unsigned out_eta = 0; int out_computed = 0; @@ -4536,7 +4561,11 @@ int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_o /* estimate the speed in MB/s */ if (delta_time != 0) - out_speed = (unsigned)(delta_size / MEGA / delta_time); + out_size_speed = (unsigned)(delta_size / MEGA / delta_time); + + /* estimate the speed in block/s */ + if (delta_pos != 0) + out_block_speed = (unsigned)(delta_pos / delta_time); /* estimate the cpu usage percentage */ if (delta_tick_total != 0) @@ -4556,12 +4585,13 @@ int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_o } if (state->opt.gui) { - log_tag("run:pos:%u:%u:%" PRIu64 ":%u:%u:%u:%u:%" PRIu64 "\n", blockpos, countpos, countsize, out_perc, out_eta, out_speed, out_cpu, (uint64_t)elapsed); + log_tag("run:pos:%u:%u:%" PRIu64 ":%u:%u:%u:%u:%" PRIu64 "\n", blockpos, countpos, countsize, out_perc, out_eta, out_size_speed, out_cpu, (uint64_t)elapsed); log_flush(); } else { msg_bar("%u%%, %u MB", out_perc, (unsigned)(countsize / MEGA)); if (out_computed) { - msg_bar(", %u MB/s", out_speed); + msg_bar(", %u MB/s", out_size_speed); + msg_bar(", %u block/s", out_block_speed); msg_bar(", CPU %u%%", out_cpu); msg_bar(", %u:%02u ETA", out_eta / 60, out_eta % 60); } diff --git a/cmdline/state.h b/cmdline/state.h index cbc1a55..4794758 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -261,7 +261,7 @@ void state_rehash(struct snapraid_state* state); */ #define SCRUB_AUTO -1 /**< Automatic selection. */ #define SCRUB_BAD -2 /**< Scrub only the bad blocks. */ -#define SCRUB_NEW -3 /**< Scub the new blocks. */ +#define SCRUB_NEW -3 /**< Scrub the new blocks. */ #define SCRUB_FULL -4 /**< Scrub everything. */ #define SCRUB_EVEN -5 /**< Even blocks. */ diff --git a/cmdline/stream.c b/cmdline/stream.c index b7bb2eb..f28338f 100644 --- a/cmdline/stream.c +++ b/cmdline/stream.c @@ -112,7 +112,7 @@ int sopen_multi_file(STREAM* s, unsigned i, const char* file) pathcpy(s->handle[i].path, sizeof(s->handle[i].path), file); - /* O_EXCL to be resilent ensure to always create a new file and not use a stale link to the original file */ + /* O_EXCL to be resilient ensure to always create a new file and not use a stale link to the original file */ f = open(file, O_WRONLY | O_CREAT | O_EXCL | O_BINARY | O_SEQUENTIAL, 0600); if (f == -1) { /* LCOV_EXCL_START */ @@ -363,7 +363,7 @@ int sgettok(STREAM* f, char* str, int size) break; } if (c == '\n') { - /* remove ending carrige return to support the Windows CR+LF format */ + /* remove ending carriage return to support the Windows CR+LF format */ if (i != str && i[-1] == '\r') --i; sungetc(c, f); @@ -429,7 +429,7 @@ int sgetline(STREAM* f, char* str, int size) while (1) { c = *pos++; if (c == '\n') { - /* remove ending carrige return to support the Windows CR+LF format */ + /* remove ending carriage return to support the Windows CR+LF format */ if (i != str && i[-1] == '\r') --i; --pos; @@ -455,7 +455,7 @@ int sgetline(STREAM* f, char* str, int size) /* LCOV_EXCL_STOP */ } if (c == '\n') { - /* remove ending carrige return to support the Windows CR+LF format */ + /* remove ending carriage return to support the Windows CR+LF format */ if (i != str && i[-1] == '\r') --i; sungetc(c, f); diff --git a/cmdline/stream.h b/cmdline/stream.h index 35b941f..6ec3391 100644 --- a/cmdline/stream.h +++ b/cmdline/stream.h @@ -28,7 +28,7 @@ * * It's not a constant for testing purpose. */ -unsigned STREAM_SIZE; +extern unsigned STREAM_SIZE; #define STREAM_STATE_READ 0 /**< The stream is in a normal state of read. */ #define STREAM_STATE_WRITE 1 /**< The stream is in a normal state of write. */ diff --git a/cmdline/support.c b/cmdline/support.c index 80dfa5c..16d0229 100644 --- a/cmdline/support.c +++ b/cmdline/support.c @@ -79,6 +79,7 @@ void lock_done(void) /* print */ int msg_level = 0; +FILE* stdlog = 0; /* * Note that in the following functions we always flush both @@ -811,7 +812,7 @@ int mkancestor(const char* file) } #ifdef _WIN32 - /* if it's a drive specificaion like "C:" */ + /* if it's a drive specification like "C:" */ if (isalpha(dir[0]) && dir[1] == ':' && dir[2] == 0) { /* nothing more to do */ return 0; @@ -1236,7 +1237,7 @@ int advise_read(struct advise_struct* advise, int f, data_off_t offset, data_off * non-blocking and do this work in a workqueue (or via some kind of * callback/continuation scheme). My worry is just doing this if a user * application does something crazy, like request gigabytes and gigabytes - * of readahead, and then repents of their craziness, there should be a + * of readahead, and then repented of their craziness, there should be a * way of cancelling the readahead request. Today, the user can just * kill the application. But if we simply shove the work to a kernel * thread, it becomes a lot harder to cancel the readahead request. We'd @@ -1632,16 +1633,16 @@ void thread_cond_wait(pthread_cond_t* cond, pthread_mutex_t* mutex) /** * Implementation note about conditional variables. * - * The conditional variables can be signaled inside or ouside the mutex, - * what is better it's debatable but in general doing that ouside the mutex, + * The conditional variables can be signaled inside or outside the mutex, + * what is better it's debatable but in general doing that outside the mutex, * reduces the number of context switches. * - * But when when testing with helgrind and drd, this disallows such tools to + * But when testing with helgrind and drd, this disallows such tools to * to see the dependency between the signal and the wait. * * To avoid it we signal everything inside the mutex. And we do this in both - * test mode (with CHERCKER defined) and release mode (CHECKER not defined), - * to be on the safe side and avoid any difference in beaviour between test and + * test mode (with CHECKER defined) and release mode (CHECKER not defined), + * to be on the safe side and avoid any difference in behaviour between test and * release. * * Here some interesting discussion: diff --git a/cmdline/support.h b/cmdline/support.h index 3b75b38..e7e76b5 100644 --- a/cmdline/support.h +++ b/cmdline/support.h @@ -109,7 +109,7 @@ void log_flush(void); /** * Pointer to log function. */ -typedef void fptr(const char* format, ...); +typedef void fptr(const char* format, ...) __attribute__((format(attribute_printf, 1, 2))); /****************************************************************************/ /* message */ @@ -415,7 +415,7 @@ int smartctl_flush(FILE* f, const char* file, const char* name); * * Ensure to change that before starting any thread. */ -int thread_cond_signal_outside; +extern int thread_cond_signal_outside; /** * Thread wrappers to handle error conditions. diff --git a/cmdline/sync.c b/cmdline/sync.c index 0d12590..96a23c8 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -1016,7 +1016,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari } /* if we have only silent errors we can try to fix them on-the-fly */ - /* note the the fix is not written to disk, but used only to */ + /* note the fix is not written to disk, but used only to */ /* compute the new parity */ if (!error_on_this_block && !io_error_on_this_block && silent_error_on_this_block) { unsigned failed_mac; @@ -1536,7 +1536,7 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t data_off_t out_size; parity_size(&parity_handle[l], &out_size); parity_overflow(state, out_size); - log_fatal("WARNING! Without an usable %s file, it isn't possible to sync.\n", lev_name(l)); + log_fatal("WARNING! Without a usable %s file, it isn't possible to sync.\n", lev_name(l)); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } diff --git a/cmdline/unix.c b/cmdline/unix.c index 351cfba..2b897fe 100644 --- a/cmdline/unix.c +++ b/cmdline/unix.c @@ -632,7 +632,7 @@ int filephy(const char* path, uint64_t size, uint64_t* physical) /* In this way we keep them in the directory traversal order */ /* that at least keeps files in the same directory together. */ /* Note also that in newer file-system with snapshot, like ZFS, */ - /* the inode doesn't represent evenmore the disk position, because files */ + /* the inode doesn't represent even more the disk position, because files */ /* are not overwritten in place, but rewritten in another location */ /* of the disk. */ *physical = FILEPHY_UNREPORTED_OFFSET; diff --git a/cmdline/unix.h b/cmdline/unix.h index 8b190ce..62da33a 100644 --- a/cmdline/unix.h +++ b/cmdline/unix.h @@ -59,7 +59,7 @@ int dirent_hidden(struct dirent* dd); const char* stat_desc(struct stat* st); /** - * Return the aligment requirement for direct IO. + * Return the alignment requirement for direct IO. */ size_t direct_size(void); diff --git a/cmdline/util.c b/cmdline/util.c index 1a35b85..7de2bab 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -470,8 +470,24 @@ static inline uint64_t util_rotl64(uint64_t x, int8_t r) return (x << r) | (x >> (64 - r)); } +/* + * Rotate right. + * In x86/x64 they are optimized with a single assembler instruction. + */ +#if 0 /* unused */ +static inline uint32_t util_rotr32(uint32_t x, int8_t r) +{ + return (x >> r) | (x << (32 - r)); +} +#endif +static inline uint64_t util_rotr64(uint64_t x, int8_t r) +{ + return (x >> r) | (x << (64 - r)); +} + + /** - * Swap endianess. + * Swap endianness. * They are needed only if BigEndian. */ #if defined(__GNUC__) @@ -502,6 +518,18 @@ static inline uint64_t util_swap64(uint64_t v) } #endif +static inline uint8_t util_read8(const void* void_ptr) +{ + const uint8_t* ptr = void_ptr; + return ptr[0]; +} + +static inline uint16_t util_read16(const void* void_ptr) +{ + const uint8_t* ptr = void_ptr; + return ptr[0] + (ptr[1] << 8); +} + static inline uint32_t util_read32(const void* ptr) { uint32_t v; @@ -543,6 +571,7 @@ static inline void util_write64(void* ptr, uint64_t v) #include "murmur3.c" #include "spooky2.c" +#include "metro.c" void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t size) { @@ -553,6 +582,9 @@ void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* case HASH_SPOOKY2 : SpookyHash128(src, size, seed, digest); break; + case HASH_METRO : + MetroHash128(src, size, seed, digest); + break; default : /* LCOV_EXCL_START */ log_fatal("Internal inconsistency in hash function %u\n", kind); @@ -568,6 +600,7 @@ const char* hash_config_name(unsigned kind) case HASH_UNDEFINED : return "undefined"; case HASH_MURMUR3 : return "murmur3"; case HASH_SPOOKY2 : return "spooky2"; + case HASH_METRO : return "metro"; default : /* LCOV_EXCL_START */ return "unknown"; diff --git a/cmdline/util.h b/cmdline/util.h index 1661a9d..8717a14 100644 --- a/cmdline/util.h +++ b/cmdline/util.h @@ -161,7 +161,7 @@ static inline uint32_t crc32c_plain(uint32_t crc, const unsigned char* ptr, unsi /** * Compute the CRC-32 (Castagnoli) */ -uint32_t (*crc32c)(uint32_t crc, const unsigned char* ptr, unsigned size); +extern uint32_t (*crc32c)(uint32_t crc, const unsigned char* ptr, unsigned size); /** * Internal entry points for testing. @@ -188,6 +188,7 @@ void crc32c_init(void); #define HASH_UNDEFINED 0 #define HASH_MURMUR3 1 #define HASH_SPOOKY2 2 +#define HASH_METRO 3 /** * Compute the HASH of a memory block. diff --git a/config.h.in b/config.h.in index 2c06e18..de1555e 100644 --- a/config.h.in +++ b/config.h.in @@ -224,6 +224,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SYS_IOCTL_H +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_MKDEV_H + /* Define to 1 if you have the header file. */ #undef HAVE_SYS_MOUNT_H @@ -240,6 +243,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SYS_STAT_H +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_SYSMACROS_H + /* Define to 1 if you have the header file. */ #undef HAVE_SYS_TYPES_H @@ -258,14 +264,6 @@ /* Define to 1 if you have the `vsnprintf' function. */ #undef HAVE_VSNPRINTF -/* Define to 1 if `major', `minor', and `makedev' are declared in . - */ -#undef MAJOR_IN_MKDEV - -/* Define to 1 if `major', `minor', and `makedev' are declared in - . */ -#undef MAJOR_IN_SYSMACROS - /* Define to 1 if assertions should be disabled. */ #undef NDEBUG diff --git a/configure b/configure index 62f8255..6580ed5 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for snapraid 11.3. +# Generated by GNU Autoconf 2.69 for snapraid 11.5. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -577,8 +577,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='snapraid' PACKAGE_TARNAME='snapraid' -PACKAGE_VERSION='11.3' -PACKAGE_STRING='snapraid 11.3' +PACKAGE_VERSION='11.5' +PACKAGE_STRING='snapraid 11.5' PACKAGE_BUGREPORT='' PACKAGE_URL='http://www.snapraid.it' @@ -1304,7 +1304,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures snapraid 11.3 to adapt to many kinds of systems. +\`configure' configures snapraid 11.5 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1374,7 +1374,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of snapraid 11.3:";; + short | recursive ) echo "Configuration of snapraid 11.5:";; esac cat <<\_ACEOF @@ -1496,7 +1496,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -snapraid configure 11.3 +snapraid configure 11.5 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2102,7 +2102,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by snapraid $as_me 11.3, which was +It was created by snapraid $as_me 11.5, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2965,7 +2965,7 @@ fi # Define the identity of the package. PACKAGE='snapraid' - VERSION='11.3' + VERSION='11.5' cat >>confdefs.h <<_ACEOF @@ -4657,6 +4657,22 @@ $as_echo "no" >&6; } fi rm -f conftest* +# This the new default for gcc 10 + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${CC-cc} accepts -fno-common" >&5 +$as_echo_n "checking whether ${CC-cc} accepts -fno-common... " >&6; } +echo 'void f(){}' > conftest.c +if test -z "`${CC-cc} -c -fno-common conftest.c 2>&1`"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + CFLAGS="$CFLAGS -fno-common" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +fi +rm -f conftest* + # Check whether --enable-largefile was given. if test "${enable_largefile+set}" = set; then : @@ -5115,56 +5131,6 @@ $as_echo "#define HAVE_SYS_WAIT_H 1" >>confdefs.h fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether sys/types.h defines makedev" >&5 -$as_echo_n "checking whether sys/types.h defines makedev... " >&6; } -if ${ac_cv_header_sys_types_h_makedev+:} false; then : - $as_echo_n "(cached) " >&6 -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -int -main () -{ -return makedev(0, 0); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_header_sys_types_h_makedev=yes -else - ac_cv_header_sys_types_h_makedev=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_sys_types_h_makedev" >&5 -$as_echo "$ac_cv_header_sys_types_h_makedev" >&6; } - -if test $ac_cv_header_sys_types_h_makedev = no; then -ac_fn_c_check_header_mongrel "$LINENO" "sys/mkdev.h" "ac_cv_header_sys_mkdev_h" "$ac_includes_default" -if test "x$ac_cv_header_sys_mkdev_h" = xyes; then : - -$as_echo "#define MAJOR_IN_MKDEV 1" >>confdefs.h - -fi - - - - if test $ac_cv_header_sys_mkdev_h = no; then - ac_fn_c_check_header_mongrel "$LINENO" "sys/sysmacros.h" "ac_cv_header_sys_sysmacros_h" "$ac_includes_default" -if test "x$ac_cv_header_sys_sysmacros_h" = xyes; then : - -$as_echo "#define MAJOR_IN_SYSMACROS 1" >>confdefs.h - -fi - - - fi -fi - for ac_header in fcntl.h stddef.h stdint.h stdlib.h string.h limits.h do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` @@ -5204,7 +5170,7 @@ fi done -for ac_header in sys/file.h sys/ioctl.h sys/vfs.h sys/statfs.h sys/param.h sys/mount.h +for ac_header in sys/file.h sys/ioctl.h sys/vfs.h sys/statfs.h sys/param.h sys/mount.h sys/sysmacros.h sys/mkdev.h do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" @@ -7373,7 +7339,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by snapraid $as_me 11.3, which was +This file was extended by snapraid $as_me 11.5, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -7436,7 +7402,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -snapraid config.status 11.3 +snapraid config.status 11.5 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 6486114..713a4cd 100644 --- a/configure.ac +++ b/configure.ac @@ -17,11 +17,13 @@ AC_CHECK_PROG([SDE],[sde],[sde],[]) AC_CHECK_PROG([ADVD2],[advd2],[advd2],[]) AM_CONDITIONAL(HAVE_ADVD2, [test x"$ADVD2" != x]) -dnl Compiler option to improve stacktrace +dnl Options to improve stacktrace AC_CHECK_CC_OPT([-fno-omit-frame-pointer], CFLAGS="$CFLAGS -fno-omit-frame-pointer", []) AC_CHECK_CC_OPT([-fno-inline-functions-called-once], CFLAGS="$CFLAGS -fno-inline-functions-called-once", []) AC_CHECK_CC_OPT([-fno-inline-small-functions], CFLAGS="$CFLAGS -fno-inline-small-functions", []) AC_CHECK_CC_OPT([-rdynamic], CFLAGS="$CFLAGS -rdynamic", []) +# This the new default for gcc 10 +AC_CHECK_CC_OPT([-fno-common], CFLAGS="$CFLAGS -fno-common", []) dnl Checks for system. AC_SYS_LARGEFILE @@ -31,11 +33,10 @@ AC_HEADER_ASSERT AC_HEADER_DIRENT AC_HEADER_TIME AC_HEADER_SYS_WAIT -AC_HEADER_MAJOR AC_CHECK_HEADERS([fcntl.h stddef.h stdint.h stdlib.h string.h limits.h]) AC_CHECK_HEADERS([unistd.h getopt.h fnmatch.h io.h inttypes.h byteswap.h]) AC_CHECK_HEADERS([pthread.h math.h]) -AC_CHECK_HEADERS([sys/file.h sys/ioctl.h sys/vfs.h sys/statfs.h sys/param.h sys/mount.h]) +AC_CHECK_HEADERS([sys/file.h sys/ioctl.h sys/vfs.h sys/statfs.h sys/param.h sys/mount.h sys/sysmacros.h sys/mkdev.h]) AC_CHECK_HEADERS([linux/fiemap.h linux/fs.h mach/mach_time.h execinfo.h]) dnl Checks for typedefs, structures, and compiler characteristics. diff --git a/raid/check.c b/raid/check.c index 9bed933..d56a150 100644 --- a/raid/check.c +++ b/raid/check.c @@ -36,7 +36,7 @@ * @ip[] Vector of @nv indexes of the valid parity blocks. * The indexes start from 0. They must be in order. * @nd Number of data blocks. - * @size Size of the blocks pointed by @v. It must be a multipler of 64. + * @size Size of the blocks pointed by @v. It must be a multiplier of 64. * @v Vector of pointers to the blocks of data and parity. * It has (@nd + @ip[@nv - 1] + 1) elements. The starting elements are the * blocks for data, following with the parity blocks. diff --git a/raid/helper.h b/raid/helper.h index bf68288..a663efd 100644 --- a/raid/helper.h +++ b/raid/helper.h @@ -23,7 +23,7 @@ * * @n Number of integers currently in the vector. * @v Vector of integers already sorted. - * It must have extra space for the new elemet at the end. + * It must have extra space for the new element at the end. * @i Value to insert. */ void raid_insert(int n, int *v, int i); diff --git a/raid/internal.h b/raid/internal.h index 4465cb9..f8d3c78 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -174,7 +174,7 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv); /* * Internal naming. * - * These are intented to provide access for testing. + * These are intended to provide access for testing. */ const char *raid_gen1_tag(void); const char *raid_gen2_tag(void); @@ -263,7 +263,7 @@ static __always_inline void raid_avx_end(void) raid_sse_end(); /* reset the upper part of the ymm registers */ - /* to avoid the 70 clocks penality on the next */ + /* to avoid the 70 clocks penalty on the next */ /* xmm register use */ asm volatile ("vzeroupper" : : : "memory"); } diff --git a/raid/memory.h b/raid/memory.h index de00614..53a428a 100644 --- a/raid/memory.h +++ b/raid/memory.h @@ -45,7 +45,7 @@ * gen2 6814 [MB/s] * genz 3033 [MB/s] * - * These are the results with displacement resulting in improvments + * These are the results with displacement resulting in improvements * in the order of 20% or more: * * sse2 diff --git a/raid/raid.c b/raid/raid.c index 3052675..ebdfd17 100644 --- a/raid/raid.c +++ b/raid/raid.c @@ -63,11 +63,11 @@ * computation of triple parity using power coefficients. * * Another important property of the Cauchy matrix is that we can setup - * the first two rows with coeffients equal at the RAID5 and RAID6 approach - * decribed, resulting in a compatible extension, and requiring SSSE3 + * the first two rows with coefficients equal at the RAID5 and RAID6 approach + * described, resulting in a compatible extension, and requiring SSSE3 * or AVX2 instructions only if triple parity or beyond is used. * - * The matrix is also adjusted, multipling each row by a constant factor + * The matrix is also adjusted, multiplying each row by a constant factor * to make the first column of all 1, to optimize the computation for * the first disk. * @@ -147,7 +147,7 @@ * "raid/test/speedtest.c" program. * * For comparison, the triple parity computation using the power - * coeffients "1,2,2^-1" is only a little faster than the one based on + * coefficients "1,2,2^-1" is only a little faster than the one based on * the Cauchy matrix if SSSE3 or AVX2 is present. * * int8 int32 int64 sse2 ssse3 avx2 @@ -205,12 +205,12 @@ void raid_zero(void *zero) * All these functions give the guarantee that parities are written * in order. First parity P, then parity Q, and so on. * This allows to specify the same memory buffer for multiple parities - * knowning that you'll get the latest written one. + * knowing that you'll get the latest written one. * This characteristic is used by the raid_delta_gen() function to * avoid to damage unused parities in recovering. * * @nd Number of data blocks - * @size Size of the blocks pointed by @v. It must be a multipler of 64. + * @size Size of the blocks pointed by @v. It must be a multiplier of 64. * @v Vector of pointers to the blocks of data and parity. * It has (@nd + #parities) elements. The starting elements are the blocks * for data, following with the parity blocks. @@ -335,7 +335,7 @@ void raid_delta_gen(int nr, int *id, int *ip, int nd, size_t size, void **v) } else { /* * Unused parities are going to be rewritten with - * not significative data, becase we don't have + * not significative data, because we don't have * functions able to compute only a subset of * parities. * @@ -483,7 +483,7 @@ void raid_rec2of2_int8(int *id, int *ip, int nd, size_t size, void **vv) * The indexes start from 0. They must be in order. * @nd Number of data blocks. * @np Number of parity blocks. - * @size Size of the blocks pointed by @v. It must be a multipler of 64. + * @size Size of the blocks pointed by @v. It must be a multiplier of 64. * @v Vector of pointers to the blocks of data and parity. * It has (@nd + @np) elements. The starting elements are the blocks * for data, following with the parity blocks. diff --git a/raid/tables.c b/raid/tables.c index 4903502..7828956 100644 --- a/raid/tables.c +++ b/raid/tables.c @@ -13662,8 +13662,8 @@ const uint8_t __aligned(256) raid_gfcauchypshufb[251][4][2][16] = /** * PSHUFB tables for generic multiplication. * - * Indexes are [MULTIPLER][LH]. - * Where MULTIPLER is from 0 to 255, LH from 0 to 1. + * Indexes are [MULTIPLIER][LH]. + * Where MULTIPLIER is from 0 to 255, LH from 0 to 1. */ const uint8_t __aligned(256) raid_gfmulpshufb[256][2][16] = { diff --git a/raid/test/Makefile b/raid/test/Makefile index 229d5ad..6423fd1 100644 --- a/raid/test/Makefile +++ b/raid/test/Makefile @@ -3,7 +3,7 @@ # # selftest - Runs the same selftest and speedtest executed at the module startup. # fulltest - Runs a more extensive test that checks all the built-in functions. -# speetest - Runs a more complete speed test. +# speedtest - Runs a more complete speed test. # invtest - Runs an extensive matrix inversion test of all the 377.342.351.231 # possible square submatrices of the Cauchy matrix used. # covtest - Runs a coverage test. diff --git a/snapraid.1 b/snapraid.1 index 0fb9b7b..ce7aa9b 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -79,6 +79,9 @@ Beside the ability to recover from disk failures, other features of SnapRAID are: .PD 0 .IP \(bu +You can use disk already filled with files, without the need to +reformat them. You will access them like now. +.IP \(bu All your data is hashed to ensure data integrity and to avoid silent corruption. .IP \(bu @@ -89,8 +92,6 @@ All the data in the other disks is safe. If you accidentally delete some files in a disk, you can recover them. .IP \(bu -You can start with already filled disks. -.IP \(bu The disks can have different sizes. .IP \(bu You can add disks at any time. @@ -98,7 +99,7 @@ You can add disks at any time. It doesn\'t lock\-in your data. You can stop using SnapRAID at any time without the need to reformat or move data. .IP \(bu -To access a file, a single disk needs to spin, saving power and +To access a file, only a single disk needs to spin, saving power and producing less noise. .PD .PP @@ -686,7 +687,7 @@ run \[dq]sync\[dq]. Later modifications are not taken into account. If bad blocks were detected, their block numbers are listed. To fix them, you can use the \[dq]fix \-e\[dq] command. .PP -It also shows a graph representing the the last time each block +It also shows a graph representing the last time each block was scrubbed or synced. Scrubbed blocks are shown with \'*\', blocks synced but not yet scrubbed with \'o\'. .PP @@ -889,7 +890,7 @@ Use the filter options to select a subset of files or disks to operate on. To only fix the blocks marked bad during \[dq]sync\[dq] and \[dq]scrub\[dq], use the \-e, \-\-filter\-error option. As difference from other filter options, with this one the fixes are -applied only to files that are not modified from the the latest \[dq]sync\[dq]. +applied only to files that are not modified from the latest \[dq]sync\[dq]. .PP All the files that cannot be fixed are renamed adding the \[dq].unrecoverable\[dq] extension. diff --git a/snapraid.d b/snapraid.d index 82ccdb5..0cc6650 100644 --- a/snapraid.d +++ b/snapraid.d @@ -31,6 +31,8 @@ Description Beside the ability to recover from disk failures, other features of SnapRAID are: + * You can use disk already filled with files, without the need to + reformat them. You will access them like now. * All your data is hashed to ensure data integrity and to avoid silent corruption. * If the failed disks are too many to allow a recovery, @@ -38,12 +40,11 @@ Description All the data in the other disks is safe. * If you accidentally delete some files in a disk, you can recover them. - * You can start with already filled disks. * The disks can have different sizes. * You can add disks at any time. * It doesn't lock-in your data. You can stop using SnapRAID at any time without the need to reformat or move data. - * To access a file, a single disk needs to spin, saving power and + * To access a file, only a single disk needs to spin, saving power and producing less noise. The official site of SnapRAID is: @@ -400,7 +401,7 @@ Commands If bad blocks were detected, their block numbers are listed. To fix them, you can use the "fix -e" command. - It also shows a graph representing the the last time each block + It also shows a graph representing the last time each block was scrubbed or synced. Scrubbed blocks are shown with '*', blocks synced but not yet scrubbed with 'o'. @@ -577,7 +578,7 @@ Commands To only fix the blocks marked bad during "sync" and "scrub", use the -e, --filter-error option. As difference from other filter options, with this one the fixes are - applied only to files that are not modified from the the latest "sync". + applied only to files that are not modified from the latest "sync". All the files that cannot be fixed are renamed adding the ".unrecoverable" extension. diff --git a/snapraid.txt b/snapraid.txt index 9621e04..78cfb35 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -38,6 +38,8 @@ big files that rarely change. Beside the ability to recover from disk failures, other features of SnapRAID are: +* You can use disk already filled with files, without the need to + reformat them. You will access them like now. * All your data is hashed to ensure data integrity and to avoid silent corruption. * If the failed disks are too many to allow a recovery, @@ -45,12 +47,11 @@ features of SnapRAID are: All the data in the other disks is safe. * If you accidentally delete some files in a disk, you can recover them. -* You can start with already filled disks. * The disks can have different sizes. * You can add disks at any time. * It doesn't lock-in your data. You can stop using SnapRAID at any time without the need to reformat or move data. -* To access a file, a single disk needs to spin, saving power and +* To access a file, only a single disk needs to spin, saving power and producing less noise. The official site of SnapRAID is: @@ -426,7 +427,7 @@ run "sync". Later modifications are not taken into account. If bad blocks were detected, their block numbers are listed. To fix them, you can use the "fix -e" command. -It also shows a graph representing the the last time each block +It also shows a graph representing the last time each block was scrubbed or synced. Scrubbed blocks are shown with '*', blocks synced but not yet scrubbed with 'o'. @@ -617,7 +618,7 @@ Use the filter options to select a subset of files or disks to operate on. To only fix the blocks marked bad during "sync" and "scrub", use the -e, --filter-error option. As difference from other filter options, with this one the fixes are -applied only to files that are not modified from the the latest "sync". +applied only to files that are not modified from the latest "sync". All the files that cannot be fixed are renamed adding the ".unrecoverable" extension. diff --git a/tommyds/tommychain.h b/tommyds/tommychain.h index 12d2514..4ff3f1a 100644 --- a/tommyds/tommychain.h +++ b/tommyds/tommychain.h @@ -114,7 +114,7 @@ tommy_inline void tommy_chain_merge(tommy_chain* first, tommy_chain* second, tom /** * Merges two chains managing special degenerated cases. - * It's funtionally equivalent at tommy_chain_merge() but faster with already ordered chains. + * It's functionally equivalent at tommy_chain_merge() but faster with already ordered chains. */ tommy_inline void tommy_chain_merge_degenerated(tommy_chain* first, tommy_chain* second, tommy_compare_func* cmp) { @@ -153,7 +153,7 @@ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func* /* * Bit buckets of chains. * Each bucket contains 2^i nodes or it's empty. - * The chain at address TOMMY_BIT_MAX is an independet variable operating as "carry". + * The chain at address TOMMY_BIT_MAX is an independent variable operating as "carry". * We keep it in the same "bit" vector to avoid reports from the valgrind tool sgcheck. */ tommy_chain bit[TOMMY_SIZE_BIT + 1]; diff --git a/tommyds/tommyhash.c b/tommyds/tommyhash.c index cc7495d..71c4ef3 100644 --- a/tommyds/tommyhash.c +++ b/tommyds/tommyhash.c @@ -231,7 +231,7 @@ tommy_uint32_t tommy_strhash_u32(tommy_uint64_t init_val, const void* void_key) key += 12; } - /* for lengths that are multiplers of 12 we already have called mix */ + /* for lengths that are multipliers of 12 we already have called mix */ /* this is different than the original lookup3 and the result won't match */ tommy_final(a, b, c); diff --git a/tommyds/tommyhash.h b/tommyds/tommyhash.h index 50d595c..e9bdba0 100644 --- a/tommyds/tommyhash.h +++ b/tommyds/tommyhash.h @@ -52,7 +52,7 @@ * \param void_key Pointer to the data to hash. * \param key_len Size of the data to hash. * \note - * This function is endianess independent. + * This function is endianness independent. * \return The hash value of 32 bits. */ tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len); @@ -72,14 +72,14 @@ tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tom * \param void_key Pointer to the data to hash. * \param key_len Size of the data to hash. * \note - * This function is endianess independent. + * This function is endianness independent. * \return The hash value of 64 bits. */ tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len); /** * String hash function with a 32 bits result. - * Implementation is based on the the Robert Jenkins "lookup3" hash 32 bits version, + * Implementation is based on Robert Jenkins "lookup3" hash 32 bits version, * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle(). * * This hash is designed to handle strings with an unknown length. If you @@ -90,7 +90,7 @@ tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tom * Use 0 if not relevant. * \param void_key Pointer to the string to hash. It has to be 0 terminated. * \note - * This function is endianess independent. + * This function is endianness independent. * \return The hash value of 32 bits. */ tommy_uint32_t tommy_strhash_u32(tommy_uint64_t init_val, const void* void_key); diff --git a/tommyds/tommyhashdyn.h b/tommyds/tommyhashdyn.h index 73baba9..0ea042f 100644 --- a/tommyds/tommyhashdyn.h +++ b/tommyds/tommyhashdyn.h @@ -92,7 +92,7 @@ * * To iterate over all the elements in the hashtable with the same key, you have to * use tommy_hashdyn_bucket() and follow the tommy_node::next pointer until NULL. - * You have also to check explicitely for the key, as the bucket may contains + * You have also to check explicitly for the key, as the bucket may contains * different keys. * * \code diff --git a/tommyds/tommylist.h b/tommyds/tommylist.h index 42ef3ad..f70ad45 100644 --- a/tommyds/tommylist.h +++ b/tommyds/tommylist.h @@ -28,7 +28,7 @@ /** \file * Double linked list for collisions into hashtables. * - * This list is a double linked list mainly targetted for handling collisions + * This list is a double linked list mainly targeted for handling collisions * into an hashtables, but useable also as a generic list. * * The main feature of this list is to require only one pointer to represent the @@ -294,7 +294,7 @@ tommy_inline void tommy_list_concat(tommy_list* first, tommy_list* second) * It's a stable merge sort with O(N*log(N)) worst complexity. * It's faster on degenerated cases like partially ordered lists. * \param cmp Compare function called with two elements. - * The function should return <0 if the first element is less than the second, ==0 if equal, and >0 if greather. + * The function should return <0 if the first element is less than the second, ==0 if equal, and >0 if greater. */ void tommy_list_sort(tommy_list* list, tommy_compare_func* cmp); diff --git a/tommyds/tommytree.h b/tommyds/tommytree.h index ada12c0..e56f865 100644 --- a/tommyds/tommytree.h +++ b/tommyds/tommytree.h @@ -33,7 +33,7 @@ * * As difference than other tommy containers, duplicate elements cannot be inserted. * - * To initialize a tree you have to call tommy_tree_init() specifing a comparison + * To initialize a tree you have to call tommy_tree_init() specifying a comparison * function that will define the order in the tree. * * \code @@ -123,7 +123,7 @@ typedef struct tommy_tree_struct { /** * Initializes the tree. - * \param cmp The comparison function that defines the orderin the tree. + * \param cmp The comparison function that defines the order in the tree. */ void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp); diff --git a/tommyds/tommytypes.h b/tommyds/tommytypes.h index 0608b15..8dc206b 100644 --- a/tommyds/tommytypes.h +++ b/tommyds/tommytypes.h @@ -227,7 +227,7 @@ typedef struct tommy_node_struct { * Compare function for elements. * \param obj_a Pointer to the first object to compare. * \param obj_b Pointer to the second object to compare. - * \return <0 if the first element is less than the second, ==0 equal, >0 if greather. + * \return <0 if the first element is less than the second, ==0 equal, >0 if greater. * * This function is like the C strcmp(). *