From d9bae1a178f0f8b198ea611e874975214ad6f990 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 1 Apr 2010 20:12:15 -0400 Subject: [PATCH] diff: cache textconv output Running a textconv filter can take a long time. It's particularly bad for a large file which needs to be spooled to disk, but even for small files, the fork+exec overhead can add up for something like "git log -p". This patch uses the notes-cache mechanism to keep a fast cache of textconv output. Caches are stored in refs/notes/textconv/$x, where $x is the userdiff driver defined in gitattributes. Caching is enabled only if diff.$x.cachetextconv is true. In my test repo, on a commit with 45 jpg and avi files changed and a textconv to show their exif tags: [before] $ time git show >/dev/null real 0m13.724s user 0m12.057s sys 0m1.624s [after, first run] $ git config diff.mfo.cachetextconv true $ time git show >/dev/null real 0m14.252s user 0m12.197s sys 0m1.800s [after, subsequent runs] $ time git show >/dev/null real 0m0.352s user 0m0.148s sys 0m0.200s So for a slight (3.8%) cost on the first run, we achieve an almost 40x speed up on subsequent runs. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- Documentation/gitattributes.txt | 20 ++++++ diff.c | 52 ++++++++++++--- t/t4042-diff-textconv-caching.sh | 109 +++++++++++++++++++++++++++++++ userdiff.c | 9 +++ userdiff.h | 4 ++ 5 files changed, 185 insertions(+), 9 deletions(-) create mode 100755 t/t4042-diff-textconv-caching.sh diff --git a/Documentation/gitattributes.txt b/Documentation/gitattributes.txt index d892e642ed..a8500d1772 100644 --- a/Documentation/gitattributes.txt +++ b/Documentation/gitattributes.txt @@ -414,6 +414,26 @@ because it quickly conveys the changes you have made), you should generate it separately and send it as a comment _in addition to_ the usual binary diff that you might send. +Because text conversion can be slow, especially when doing a +large number of them with `git log -p`, git provides a mechanism +to cache the output and use it in future diffs. To enable +caching, set the "cachetextconv" variable in your diff driver's +config. For example: + +------------------------ +[diff "jpg"] + textconv = exif + cachetextconv = true +------------------------ + +This will cache the result of running "exif" on each blob +indefinitely. If you change the textconv config variable for a +diff driver, git will automatically invalidate the cache entries +and re-run the textconv filter. If you want to invalidate the +cache manually (e.g., because your version of "exif" was updated +and now produces better output), you can remove the cache +manually with `git update-ref -d refs/notes/textconv/jpg` (where +"jpg" is the name of the diff driver, as in the example above). Performing a three-way merge ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/diff.c b/diff.c index 9665d6d41f..72d8503d89 100644 --- a/diff.c +++ b/diff.c @@ -43,7 +43,7 @@ static char diff_colors[][COLOR_MAXLEN] = { }; static void diff_filespec_load_driver(struct diff_filespec *one); -static size_t fill_textconv(const char *cmd, +static size_t fill_textconv(struct userdiff_driver *driver, struct diff_filespec *df, char **outbuf); static int parse_diff_color_slot(const char *var, int ofs) @@ -466,8 +466,8 @@ static void emit_rewrite_diff(const char *name_a, const char *name_b, struct diff_filespec *one, struct diff_filespec *two, - const char *textconv_one, - const char *textconv_two, + struct userdiff_driver *textconv_one, + struct userdiff_driver *textconv_two, struct diff_options *o) { int lc_a, lc_b; @@ -1569,14 +1569,26 @@ void diff_set_mnemonic_prefix(struct diff_options *options, const char *a, const options->b_prefix = b; } -static const char *get_textconv(struct diff_filespec *one) +static struct userdiff_driver *get_textconv(struct diff_filespec *one) { if (!DIFF_FILE_VALID(one)) return NULL; if (!S_ISREG(one->mode)) return NULL; diff_filespec_load_driver(one); - return one->driver->textconv; + if (!one->driver->textconv) + return NULL; + + if (one->driver->textconv_want_cache && !one->driver->textconv_cache) { + struct notes_cache *c = xmalloc(sizeof(*c)); + struct strbuf name = STRBUF_INIT; + + strbuf_addf(&name, "textconv/%s", one->driver->name); + notes_cache_init(c, name.buf, one->driver->textconv); + one->driver->textconv_cache = c; + } + + return one->driver; } static void builtin_diff(const char *name_a, @@ -1593,7 +1605,8 @@ static void builtin_diff(const char *name_a, const char *set = diff_get_color_opt(o, DIFF_METAINFO); const char *reset = diff_get_color_opt(o, DIFF_RESET); const char *a_prefix, *b_prefix; - const char *textconv_one = NULL, *textconv_two = NULL; + struct userdiff_driver *textconv_one = NULL; + struct userdiff_driver *textconv_two = NULL; struct strbuf header = STRBUF_INIT; if (DIFF_OPT_TST(o, SUBMODULE_LOG) && @@ -3888,13 +3901,13 @@ static char *run_textconv(const char *pgm, struct diff_filespec *spec, return strbuf_detach(&buf, outsize); } -static size_t fill_textconv(const char *cmd, +static size_t fill_textconv(struct userdiff_driver *driver, struct diff_filespec *df, char **outbuf) { size_t size; - if (!cmd) { + if (!driver || !driver->textconv) { if (!DIFF_FILE_VALID(df)) { *outbuf = ""; return 0; @@ -3905,8 +3918,29 @@ static size_t fill_textconv(const char *cmd, return df->size; } - *outbuf = run_textconv(cmd, df, &size); + if (driver->textconv_cache) { + *outbuf = notes_cache_get(driver->textconv_cache, df->sha1, + &size); + if (*outbuf) + return size; + } + + *outbuf = run_textconv(driver->textconv, df, &size); if (!*outbuf) die("unable to read files to diff"); + + if (driver->textconv_cache) { + /* ignore errors, as we might be in a readonly repository */ + notes_cache_put(driver->textconv_cache, df->sha1, *outbuf, + size); + /* + * we could save up changes and flush them all at the end, + * but we would need an extra call after all diffing is done. + * Since generating a cache entry is the slow path anyway, + * this extra overhead probably isn't a big deal. + */ + notes_cache_write(driver->textconv_cache); + } + return size; } diff --git a/t/t4042-diff-textconv-caching.sh b/t/t4042-diff-textconv-caching.sh new file mode 100755 index 0000000000..91f8198f05 --- /dev/null +++ b/t/t4042-diff-textconv-caching.sh @@ -0,0 +1,109 @@ +#!/bin/sh + +test_description='test textconv caching' +. ./test-lib.sh + +cat >helper <<'EOF' +#!/bin/sh +sed 's/^/converted: /' "$@" >helper.out +cat helper.out +EOF +chmod +x helper + +test_expect_success 'setup' ' + echo foo content 1 >foo.bin && + echo bar content 1 >bar.bin && + git add . && + git commit -m one && + echo foo content 2 >foo.bin && + echo bar content 2 >bar.bin && + git commit -a -m two && + echo "*.bin diff=magic" >.gitattributes && + git config diff.magic.textconv ./helper && + git config diff.magic.cachetextconv true +' + +cat >expect <actual && + test_cmp expect actual +' + +test_expect_success 'cached textconv produces same output' ' + git diff HEAD^ HEAD >actual && + test_cmp expect actual +' + +test_expect_success 'cached textconv does not run helper' ' + rm -f helper.out && + git diff HEAD^ HEAD >actual && + test_cmp expect actual && + ! test -r helper.out +' + +cat >expect <other && + git config diff.magic.textconv "./helper other" && + git diff HEAD^ HEAD >actual && + test_cmp expect actual +' + +cat >expect <>.gitattributes && + git diff HEAD^ HEAD >actual && + test_cmp expect actual +' + +test_done diff --git a/userdiff.c b/userdiff.c index df992490d5..67003fbb23 100644 --- a/userdiff.c +++ b/userdiff.c @@ -1,3 +1,4 @@ +#include "cache.h" #include "userdiff.h" #include "cache.h" #include "attr.h" @@ -167,6 +168,12 @@ static int parse_tristate(int *b, const char *k, const char *v) return 1; } +static int parse_bool(int *b, const char *k, const char *v) +{ + *b = git_config_bool(k, v); + return 1; +} + int userdiff_config(const char *k, const char *v) { struct userdiff_driver *drv; @@ -181,6 +188,8 @@ int userdiff_config(const char *k, const char *v) return parse_string(&drv->external, k, v); if ((drv = parse_driver(k, v, "textconv"))) return parse_string(&drv->textconv, k, v); + if ((drv = parse_driver(k, v, "cachetextconv"))) + return parse_bool(&drv->textconv_want_cache, k, v); if ((drv = parse_driver(k, v, "wordregex"))) return parse_string(&drv->word_regex, k, v); diff --git a/userdiff.h b/userdiff.h index c3151594f5..942d594950 100644 --- a/userdiff.h +++ b/userdiff.h @@ -1,6 +1,8 @@ #ifndef USERDIFF_H #define USERDIFF_H +#include "notes-cache.h" + struct userdiff_funcname { const char *pattern; int cflags; @@ -13,6 +15,8 @@ struct userdiff_driver { struct userdiff_funcname funcname; const char *word_regex; const char *textconv; + struct notes_cache *textconv_cache; + int textconv_want_cache; }; int userdiff_config(const char *k, const char *v);