Implemented recovery of updates committed by Windows

ntfsrecover applies to the metadata the updates which were requested on
Windows but could not be completed because they were interrupted by
some event such as a power failure, a hardware crash, a software crash
or the device being unplugged. Doing so, the file system is restored
to the latest consistent state.

No update to libntfs-3g is required by this implementation.

configure.ac

@@ -657,6 +657,7 @@ AC_CONFIG_FILES([
 	ntfsprogs/ntfswipe.8
 	ntfsprogs/ntfstruncate.8
 	ntfsprogs/ntfsfallocate.8
+	ntfsprogs/ntfsrecover.8
 	src/Makefile
 	src/ntfs-3g.8
 	src/ntfs-3g.probe.8

ntfsprogs/Makefile.am

@@ -17,7 +17,7 @@ if ENABLE_NTFSPROGS
 bin_PROGRAMS		= ntfsfix ntfsinfo ntfscluster ntfsls ntfscat ntfscmp
 sbin_PROGRAMS		= mkntfs ntfslabel ntfsundelete ntfsresize ntfsclone \
 			  ntfscp
-EXTRA_PROGRAM_NAMES	= ntfswipe ntfstruncate
+EXTRA_PROGRAM_NAMES	= ntfswipe ntfstruncate ntfsrecover
 
 QUARANTINED_PROGRAM_NAMES = ntfsdump_logfile ntfsmftalloc ntfsmove ntfsck \
 			   ntfsfallocate
@@ -26,7 +26,7 @@ man_MANS		= mkntfs.8 ntfsfix.8 ntfslabel.8 ntfsinfo.8 \
 			  ntfsundelete.8 ntfsresize.8 ntfsprogs.8 ntfsls.8 \
 			  ntfsclone.8 ntfscluster.8 ntfscat.8 ntfscp.8 \
 			  ntfscmp.8 ntfswipe.8 ntfstruncate.8 \
-			  ntfsdecrypt.8 ntfsfallocate.8
+			  ntfsdecrypt.8 ntfsfallocate.8 ntfsrecover.8
 EXTRA_MANS		=
 
 CLEANFILES		= $(EXTRA_PROGRAMS)
@@ -102,6 +102,10 @@ ntfscmp_SOURCES		= ntfscmp.c utils.c utils.h
 ntfscmp_LDADD		= $(AM_LIBS)
 ntfscmp_LDFLAGS		= $(AM_LFLAGS)
 
+ntfsrecover_SOURCES	= playlog.c ntfsrecover.c utils.c utils.h ntfsrecover.h
+ntfsrecover_LDADD	= $(AM_LIBS) $(NTFSRECOVER_LIBS)
+ntfsrecover_LDFLAGS	= $(AM_LFLAGS)
+
 # We don't distribute these
 
 ntfstruncate_SOURCES	= attrdef.c ntfstruncate.c utils.c utils.h

ntfsprogs/ntfsrecover.8.in

@@ -0,0 +1,165 @@
+.\" Copyright (c) 2015 Jean-Pierre Andre
+.\" This file may be copied under the terms of the GNU Public License.
+.\"
+.TH NTFSRECOVER 8 "September 2015" "ntfs-3g @VERSION@"
+.SH NAME
+ntfsrecover \- Recover updates committed by Windows on an NTFS volume
+.SH SYNOPSIS
+\fBntfsrecover\fR [\fIoptions\fR] \fIdevice\fR
+.SH DESCRIPTION
+.B ntfsrecover
+applies to the metadata the updates which were requested on Windows but could
+not be completed because they were interrupted by some event such as a power
+failure, a hardware crash, a software crash or the device being unplugged.
+Doing so, the file system is restored to a consistent state, however updates
+to user data may still be lost.
+
+Updating the file system generally requires updating several records which
+should all be made for the file system to be kept consistent. For instance,
+creating a new file requires reserving an inode number (set a bit in a bit
+map), creating a file record (store the file name and file attributes), and
+registering the file in a directory (locate the file from some path). When an
+unfortunate event occurs, and one of these updates could be done but not all
+of them, the file system is left inconsistent.
+
+A group of updates which have all to be done to preserve consistency is
+called a transaction, and the end of updates within a transaction is called
+the commitment of the transaction.
+
+To protect from unfortunate events, Windows first logs in a special file all
+the metadata update requests without applying any, until the commitment is
+known. If the event occurs before the commitment, no update has been made and
+the file system is consistent. If the event occurs after the update, the log
+file can be analyzed later and the transactions which were committed can be
+executed again, thus restoring the integrity of the file system.
+
+.B ntfsrecover
+similarly examines the log file and applies the updates within committed
+transactions which could not be done by Windows.
+
+Currently, ntfs-3g does not log updates, so
+.B ntfsrecover
+cannot be used to restore consistency after an unfortunate event occurred
+while the file system was updated by Linux.
+
+.SH OPTIONS
+Below is a summary of all the options that
+.B ntfsrecover
+accepts. The normal usage is to use no option at all, as most of these
+options are oriented towards developers needs.
+
+Nearly all options have two equivalent names.  The short name is
+preceded by
+.B \-
+and the long name is preceded by
+.BR \-\- .
+Any single letter options, that don't take an argument, can be combined into a
+single command, e.g.
+.B \-bv
+is equivalent to
+.BR "\-b \-v" .
+Long named options can be abbreviated to any unique prefix of their name.
+.TP
+\fB\-b\fR, \fB\-\-backward\fR
+Examine the actions described in the logfile backward from the latest one to
+the earliest one without applying any update. This may encompass records
+generated during several sessions, and when Windows is restarted, it often
+does not restart writing where it ended the previous session, so this leads
+to errors and bad sequencing when examining the full log file.
+.TP
+\fB\-c\fR, \fB\-\-clusters\fR \fBCLUSTER-RANGE\fR
+Restrict the output generated when using options -b -f -u -p
+to the actions operating on a cluster within the given cluster range.
+CLUSTER-RANGE is defined by the first and last cluster numbers separated
+by a hyphen, for instance 100-109 or 0x3e8-0x3ff. A single number means
+restricting to a single cluster. The first four log blocks have a special
+role and they are always shown.
+.TP
+\fB\-f\fR, \fB\-\-forward\fR \fBNUM\fR
+Examine the actions described in the logfile forward from the first one to
+the last one without applying any update. As the log file is reused
+circularly, the first one is generally not the earliest. Moreover when
+Windows is restarted, it often does not restart writing where it ended the
+previous sessions, and this leads to errors when examining a log file
+generated during several sessions.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+Show some help information.
+.TP
+\fB\-n\fR, \fB\-\-no-action\fR
+Do not apply any modification, useful when using the options -p, -s or -u.
+.TP
+\fB\-p\fR, \fB\-\-play\fR \fBCOUNT\fR
+Undo COUNT transaction sets and redo a single one, a transaction set being
+all transactions between two consecutive checkpoints. This is useful for
+replaying some transaction in the past. As a few actions are not undoable,
+this is not always possible.
+.TP
+\fB\-r\fR, \fB\-\-range\fR \fBBLOCK-RANGE\fR
+Examine the actions described in the logfile forward restricted to the
+requested log file block range without applying any update. The first four
+log blocks have a special role and they are always examined.
+.TP
+\fB\-s\fR, \fB\-\-sync\fR
+Sync the file system by applying the committed actions which have not
+been synced previously. This is the default option, used when none of
+the options -n, -f, -r, -p and -u are present.
+
+The option -s can be repeated to request applying the committed actions
+mentioned in the obsolete restart page. This is useful for testing the
+situations where the latest restart page cannot be read though it can
+actually be read.
+.TP
+\fB\-t\fR, \fB\-\-transactions\fR \fBCOUNT\fR
+Display the transaction parameters when examining the log file with one
+of the options --forward, --backward or --range.
+.TP
+\fB\-u\fR, \fB\-\-undo\fR \fBCOUNT\fR
+Undo COUNT transaction sets, thus resetting the file system to some
+checkpoint in the past, a transaction set being all transactions between
+two consecutive checkpoints. As a few actions are not undoable, this is
+not always possible.
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+Display more debug/warning/error messages. This option may be used twice
+to display even more information.
+.TP
+\fB\-V\fR, \fB\-\-version\fR
+Show the version number, copyright and license of
+.BR ntfsrecover .
+.SH EXAMPLES
+Sync an NTFS volume on /dev/sda1.
+.RS
+.sp
+.B ntfsrecover -s /dev/sda1
+.sp
+.RE
+Display all actions which updated a cluster in range 100 to 119 :
+.RS
+.sp
+.B ntfsrecover --verbose --backward --clusters=100-119 /dev/sda1
+.sp
+.RE
+.SH BUGS
+If you find a bug please send an email describing the problem to the
+development team:
+.br
+.nh
+ntfs\-3g\-devel@lists.sf.net
+.hy
+.SH AUTHORS
+.B ntfsrecover
+was written by Jean-Pierre Andre
+.SH AVAILABILITY
+.B ntfsrecover
+is part of the
+.B ntfs-3g
+package and is available from:
+.br
+.nh
+http://www.tuxera.com/community/
+.hy
+.SH SEE ALSO
+.BR ntfs-3g (8),
+.BR ntfsfix (8),
+.BR ntfsprogs (8)

ntfsprogs/ntfsrecover.h

@@ -0,0 +1,339 @@
+/*
+ *		Declarations for processing log data
+ *
+ * Copyright (c) 2000-2005 Anton Altaparmakov
+ * Copyright (c) 2014-2015 Jean-Pierre Andre
+ */
+
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program (in the main directory of the NTFS-3G
+ * distribution in the file COPYING); if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+/*
+ * TODO
+ *	This file partially duplicates logfile.h (with modifications).
+ *	The generic declarations are to be moved to logfile.h, thus
+ *	implying adapting (at least) libntfs-3g/logfile.c and
+ *	ntfsprogs/ntfsdump_logfile.c, and the declarations specific to
+ *	ntfsrecover should be kept in this file.
+ *	(removing ntfsdump_logfile.c might also be considered).
+ */
+
+#define getle16(p,x) le16_to_cpu(*(const le16*)((const char*)(p) + (x)))
+#define getle32(p,x) le32_to_cpu(*(const le32*)((const char*)(p) + (x)))
+#define getle64(p,x) le64_to_cpu(*(const le64*)((const char*)(p) + (x)))
+
+#define feedle16(p,x) (*(const le16*)((const char*)(p) + (x)))
+#define feedle32(p,x) (*(const le32*)((const char*)(p) + (x)))
+#define feedle64(p,x) (*(const le64*)((const char*)(p) + (x)))
+
+enum LOG_RECORD_TYPE {
+	LOG_STANDARD = 1,
+	LOG_CHECKPOINT = 2
+} ;
+
+	/* These flags were introduced in Vista in field attribute_flags */
+enum ATTRIBUTE_FLAGS {
+	ACTS_ON_MFT = 2,
+	ACTS_ON_INDX = 8
+} ;
+
+enum ACTIONS {
+	Noop,					/* 0 */
+	CompensationlogRecord,			/* 1 */
+	InitializeFileRecordSegment,		/* 2 */
+	DeallocateFileRecordSegment,		/* 3 */
+	WriteEndofFileRecordSegment,		/* 4 */
+	CreateAttribute,			/* 5 */
+	DeleteAttribute,			/* 6 */
+	UpdateResidentValue,			/* 7 */
+	UpdateNonResidentValue,			/* 8 */
+	UpdateMappingPairs,			/* 9 */
+	DeleteDirtyClusters,			/* 10 */
+	SetNewAttributeSizes,			/* 11 */
+	AddIndexEntryRoot,			/* 12 */
+	DeleteIndexEntryRoot,			/* 13 */
+	AddIndexEntryAllocation,		/* 14 */
+	DeleteIndexEntryAllocation,		/* 15 */
+	WriteEndOfIndexBuffer,			/* 16 */
+	SetIndexEntryVcnRoot,			/* 17 */
+	SetIndexEntryVcnAllocation,		/* 18 */
+	UpdateFileNameRoot,			/* 19 */
+	UpdateFileNameAllocation,		/* 20 */
+	SetBitsInNonResidentBitMap,		/* 21 */
+	ClearBitsInNonResidentBitMap,		/* 22 */
+	HotFix,					/* 23 */
+	EndTopLevelAction,			/* 24 */
+	PrepareTransaction,			/* 25 */
+	CommitTransaction,			/* 26 */
+	ForgetTransaction,			/* 27 */
+	OpenNonResidentAttribute,		/* 28 */
+	OpenAttributeTableDump,			/* 29 */
+	AttributeNamesDump,			/* 30 */
+	DirtyPageTableDump,			/* 31 */
+	TransactionTableDump,			/* 32 */
+	UpdateRecordDataRoot,			/* 33 */
+	UpdateRecordDataAllocation,		/* 34 */
+	Win10Action35,				/* 35 */
+	Win10Action36,				/* 36 */
+	Win10Action37,				/* 37 */
+	LastAction				/* 38 */
+} ;
+
+	/* Flags for field log_record_flags, their meaning is unclear */
+enum RECORD_FLAGS {
+	RECORD_UNKNOWN = 1,
+	/* The flags below were introduced in Windows 10 */
+	RECORD_DELETING = 2,
+	RECORD_ADDING = 4
+} ;
+typedef le16 LOG_RECORD_FLAGS;
+
+#define LOGFILE_NO_CLIENT const_cpu_to_le16(0xffff)
+#define RESTART_VOLUME_IS_CLEAN const_cpu_to_le16(0x0002)
+
+/* ntfsdoc p 39 (47), not in layout.h */
+
+typedef struct RESTART_PAGE_HEADER { /* size 32 */
+	NTFS_RECORD head;
+	le64 chkdsk_lsn;
+	le32 system_page_size;
+	le32 log_page_size;
+	le16 restart_offset;
+	le16 minor_ver;
+	le16 major_ver;
+	le16 usn;
+} __attribute__((__packed__)) RESTART_PAGE_HEADER;
+
+/* ntfsdoc p 40 (48), not in layout.h */
+
+struct RESTART_AREA { /* size 44 */
+	le64 current_lsn;
+	le16 log_clients;
+	le16 client_free_list;
+	le16 client_in_use_list;
+	le16 flags;
+	le32 seq_number_bits;
+	le16 restart_area_length;
+	le16 client_array_offset;
+	le64 file_size;
+	le32 last_lsn_data_length;
+	le16 record_length;
+	le16 log_page_data_offset;
+	le32 restart_log_open_count;
+} __attribute__((__packed__)) ;
+
+typedef struct RESTART_CLIENT { /* size 160 */
+/*Ofs*/
+/*  0*/	le64 oldest_lsn;	/* Oldest LSN needed by this client.  On create
+				   set to 0. */
+/*  8*/	le64 client_restart_lsn;/* LSN at which this client needs to restart
+				   the volume, i.e. the current position within
+				   the log file.  At present, if clean this
+				   should = current_lsn in restart area but it
+				   probably also = current_lsn when dirty most
+				   of the time.  At create set to 0. */
+/* 16*/	le16 prev_client;	/* The offset to the previous log client record
+				   in the array of log client records.
+				   LOGFILE_NO_CLIENT means there is no previous
+				   client record, i.e. this is the first one.
+				   This is always LOGFILE_NO_CLIENT. */
+/* 18*/	le16 next_client;	/* The offset to the next log client record in
+				   the array of log client records.
+				   LOGFILE_NO_CLIENT means there are no next
+				   client records, i.e. this is the last one.
+				   This is always LOGFILE_NO_CLIENT. */
+/* 20*/	le16 seq_number;	/* On Win2k and presumably earlier, this is set
+				   to zero every time the logfile is restarted
+				   and it is incremented when the logfile is
+				   closed at dismount time.  Thus it is 0 when
+				   dirty and 1 when clean.  On WinXP and
+				   presumably later, this is always 0. */
+/* 22*/	u8 reserved[6];		/* Reserved/alignment. */
+/* 28*/	le32 client_name_length;/* Length of client name in bytes.  Should
+				   always be 8. */
+/* 32*/	le16 client_name[64];   /* Name of the client in Unicode.  Should
+				   always be "NTFS" with the remaining bytes
+				   set to 0. */
+/* sizeof() = 160 (0xa0) bytes */
+} __attribute__((__packed__)) LOG_CLIENT_RECORD;
+
+/* ntfsdoc p 41 (49), not in layout.h */
+
+struct RECORD_PAGE_HEADER { /* size 40 */
+	NTFS_RECORD head;       /* the magic is "RCRD" */
+	union {
+		le64 last_lsn;
+		le32 file_offset;
+	} __attribute__((__packed__)) copy;
+	le32 flags;
+	le16 page_count;
+	le16 page_position;
+	le16 next_record_offset;
+	le16 reserved4[3];
+	le64 last_end_lsn;
+} __attribute__((__packed__)) ;
+
+/* ntfsdoc p 42 (50), not in layout.h */
+
+#define LOG_RECORD_HEAD_SZ 0x30 /* size of header of struct LOG_RECORD */
+
+typedef struct LOG_RECORD { /* size 80 */
+	le64 this_lsn;
+	le64 client_previous_lsn;
+	le64 client_undo_next_lsn;
+	le32 client_data_length;
+	struct {
+		le16 seq_number;
+		le16 client_index;
+	} __attribute__((__packed__)) client_id;
+	le32 record_type;
+	le32 transaction_id;
+	LOG_RECORD_FLAGS log_record_flags;
+	le16 reserved1[3];
+	le16 redo_operation;
+	le16 undo_operation;
+	le16 redo_offset;
+	le16 redo_length;
+	union {
+		struct {
+			le16 undo_offset;
+			le16 undo_length;
+			le16 target_attribute;
+			le16 lcns_to_follow;
+			le16 record_offset;
+			le16 attribute_offset;
+			le16 cluster_index;
+			le16 attribute_flags;
+			le32 target_vcn;
+			le32 reserved3;
+			le64 lcn_list[0];
+		} __attribute__((__packed__));
+		struct {
+			le64 transaction_lsn;
+			le64 attributes_lsn;
+			le64 names_lsn;
+			le64 dirty_pages_lsn;
+			le64 unknown_list[0];
+		} __attribute__((__packed__));
+	} __attribute__((__packed__));
+} __attribute__((__packed__)) LOG_RECORD;
+
+struct BUFFER {
+	unsigned int num;
+	unsigned int size;
+	unsigned int headsz;
+	BOOL safe;
+	union {
+		struct RESTART_PAGE_HEADER restart;
+		struct RECORD_PAGE_HEADER record;
+		char data[1];
+	} block;  /* variable length, keep at the end */
+} ;
+
+struct ACTION_RECORD {
+	struct ACTION_RECORD *next;
+	struct ACTION_RECORD *prev;
+	int num;
+	unsigned int flags;
+	struct LOG_RECORD record; /* variable length, keep at the end */
+} ;
+
+enum {		/* Flag values for ACTION_RECORD */
+	ACTION_TO_REDO = 1	/* Committed, possibly not synced */
+	} ;
+
+struct ATTR {
+	u64 inode;
+	u64 lsn;
+	le32 type;
+	u16 key;
+	u16 namelen;
+	le16 name[1];
+} ;
+
+struct BITMAP_ACTION {
+	le32 firstbit;
+	le32 count;
+} ;
+
+/* Danger in arrays : contains le64's though size is not a multiple of 8 */
+typedef struct ATTR_OLD {	/* Format up to Win10 (44 bytes) */
+	le64 unknown1;
+	le64 unknown2;
+	le64 inode;
+	le64 lsn;
+	le32 unknown3;
+	le32 type;
+	le32 unknown4;
+} __attribute__((__packed__)) ATTR_OLD;
+
+typedef struct ATTR_NEW {	/* Format since Win10 (40 bytes) */
+	le64 unknown1;
+	le64 unknown2;
+	le32 type;
+	le32 unknown3;
+	le64 inode;
+	le64 lsn;
+} __attribute__((__packed__)) ATTR_NEW;
+
+extern u32 clustersz;
+extern int clusterbits;
+extern u32 blocksz;
+extern int blockbits;
+extern u16 bytespersect;
+extern u64 mftlcn;
+extern u32 mftrecsz;
+extern int mftrecbits;
+extern u32 mftcnt; /* number of entries */
+extern BOOL optc;
+extern BOOL optn;
+extern int opts;
+extern int optv;
+extern unsigned int redocount;
+extern unsigned int undocount;
+extern ntfs_inode *log_ni;
+extern ntfs_attr *log_na;
+extern u64 logfilelcn;
+extern u32 logfilesz; /* bytes */
+extern u64 redos_met;
+extern u64 committed_lsn;
+extern u64 synced_lsn;
+extern u64 latest_lsn;
+extern u64 restart_lsn;
+
+extern struct RESTART_AREA restart;
+extern struct RESTART_CLIENT client;
+
+const char *actionname(int op);
+const char *mftattrname(ATTR_TYPES attr);
+void showname(const char *prefix, const char *name, int cnt);
+int fixnamelen(const char *name, int len);
+BOOL within_lcn_range(const struct LOG_RECORD *logr);
+struct ATTR *getattrentry(unsigned int key, unsigned int lth);
+void copy_attribute(struct ATTR *pa, const char *buf, int length);
+u32 get_undo_offset(const struct LOG_RECORD *logr);
+u32 get_redo_offset(const struct LOG_RECORD *logr);
+u32 get_extra_offset(const struct LOG_RECORD *logr);
+BOOL exception(int num);
+
+struct STORE;
+BOOL ntfs_check_logfile(ntfs_attr *log_na, RESTART_PAGE_HEADER **rp);
+extern int play_undos(ntfs_volume *vol, const struct ACTION_RECORD *firstundo);
+extern int play_redos(ntfs_volume *vol, const struct ACTION_RECORD *firstredo);
+extern void show_redos(void);
+extern void freeclusterentry(struct STORE*);
+void hexdump(const char *buf, unsigned int lth);