diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile index b36a1dd97c..9fd819b398 100644 --- a/libraries/liblmdb/Makefile +++ b/libraries/liblmdb/Makefile @@ -112,7 +112,11 @@ mtest6: mtest6.o liblmdb.a mtest_remap: mtest_remap.o liblmdb.a mtest_enc: mtest_enc.o chacha8.o liblmdb.a mtest_enc2: mtest_enc2.o liblmdb.a crypto.lm - $(CC) $(LDFLAGS) -pthread -o $@ mtest_enc2.o liblmdb.a $(LDL) + $(CC) $(LDFLAGS) -pthread -o $@ $@.o liblmdb.a $(LDL) +mtest_enc3: mtest_enc3.o liblmdb.a crypto.lm + $(CC) $(LDFLAGS) -pthread -o $@ $@.o liblmdb.a $(LDL) +mtest_enc4: mtest_enc4.o liblmdb.a crypto.lm + $(CC) $(LDFLAGS) -pthread -o $@ $@.o liblmdb.a $(LDL) mplay: mplay.o liblmdb.a diff --git a/libraries/liblmdb/crypto.c b/libraries/liblmdb/crypto.c index 14fc737c83..a0fa58a38c 100644 --- a/libraries/liblmdb/crypto.c +++ b/libraries/liblmdb/crypto.c @@ -57,7 +57,15 @@ static int mcf_encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int if (encdec) { rc = ENC(dst, src, key[2].mv_data, key[0].mv_data, iv); } else { - rc = DEC(dst, src, key[2].mv_data, key[0].mv_data, iv); + if (key[2].mv_size) + rc = DEC(dst, src, key[2].mv_data, key[0].mv_data, iv); + else { + /* Decrypting without MAC check: only used when decrypting page + * headers during incremental load. + */ + rc = crypto_stream_chacha20_ietf_xor_ic(dst->mv_data, src->mv_data, + src->mv_size, iv, 1, key[0].mv_data); + } } return rc; } diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 7c099a2aa0..f2dff391ee 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -1655,7 +1655,7 @@ struct MDB_env { HANDLE me_lfd; /**< The lock file */ HANDLE me_mfd; /**< For writing and syncing the meta pages */ #ifdef _WIN32 -#ifdef MDB_RPAGE_CACHE +#if MDB_RPAGE_CACHE HANDLE me_fmh; /**< File Mapping handle */ #endif HANDLE me_ovfd; /**< Overlapped/async with write-through file handle */ @@ -5527,7 +5527,7 @@ mdb_env_open2(MDB_env *env, int prev) env->me_psize = meta.mm_psize; } env->me_pagespace = env->me_psize - PAGEHDRSZ -#ifdef MDB_RPAGE_CACHE +#if MDB_RPAGE_CACHE - env->me_sumsize - env->me_esumsize #endif ; @@ -6439,8 +6439,10 @@ mdb_env_close_active(MDB_env *env, int excl) if (env->me_rpages) { MDB_ID3L el = env->me_rpages; unsigned int x; - for (x=1; x<=el[0].mid; x++) + for (x=1; x<=el[0].mid; x++) { munmap(el[x].mptr, el[x].mcnt * env->me_psize); + free(el[x].menc); + } free(el); } } @@ -6905,10 +6907,11 @@ mdb_rpage_encsum(MDB_env *env, MDB_ID3 *id3, unsigned rem, int numpgs) * @param[in] pg0 the page number for the page to retrieve. * @param[in] numpgs number of database pages (can be > 1 for overflow pages) * @param[out] ret address of a pointer where the page's address will be stored. + * @param[out] enc address of a pointer where the encrypted page's address will be stored. * @return 0 on success, non-zero on failure. */ static int -mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret) +mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret, MDB_page **enc) { MDB_env *env = txn->mt_env; MDB_page *p; @@ -7110,8 +7113,10 @@ retry: id3.mid = pg0; if (env->me_encfunc || env->me_sumfunc) { rc = mdb_rpage_encsum(env, &id3, rem, numpgs); - if (rc) + if (rc) { + free(id3.menc); goto fail; + } el[x].muse = id3.muse; } pthread_mutex_unlock(&env->me_rpmutex); @@ -7178,8 +7183,10 @@ fail: } if (env->me_encfunc || env->me_sumfunc) { rc = mdb_rpage_encsum(env, &id3, rem, numpgs); - if (rc) + if (rc) { + free(id3.menc); goto fail; + } } mdb_mid3l_insert(el, &id3); pthread_mutex_unlock(&env->me_rpmutex); @@ -7205,6 +7212,9 @@ ok: } #endif *ret = p; + if (enc && env->me_encfunc) { + *enc = (MDB_page *)(id3.mptr + rem * env->me_psize); + } return rc; } @@ -7388,7 +7398,7 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno, mapped: #if MDB_RPAGE_CACHE if (MDB_REMAPPING(txn->mt_env->me_flags)) { - int rc = mdb_rpage_get(txn, pgno, numpgs, &p); + int rc = mdb_rpage_get(txn, pgno, numpgs, &p, NULL); if (rc) { txn->mt_flags |= MDB_TXN_ERROR; return rc; @@ -8571,10 +8581,10 @@ _mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, return MDB_BAD_VALSIZE; #if SIZE_MAX > MAXDATASIZE - if (data->mv_size > ((mc->mc_db->md_flags & MDB_DUPSORT) ? ENV_MAXKEY(env) : MAXDATASIZE)) + if (data->mv_size-1 >= ((mc->mc_db->md_flags & MDB_DUPSORT) ? ENV_MAXKEY(env) : MAXDATASIZE)) return MDB_BAD_VALSIZE; #else - if ((mc->mc_db->md_flags & MDB_DUPSORT) && data->mv_size > ENV_MAXKEY(env)) + if ((mc->mc_db->md_flags & MDB_DUPSORT) && data->mv_size-1 >= ENV_MAXKEY(env)) return MDB_BAD_VALSIZE; #endif @@ -11136,6 +11146,10 @@ mdb_env_copythr(void *arg) char *ptr; int toggle = 0, rc; mdb_size_t wsize; +#if MDB_RPAGE_CACHE + char *pagebuf = NULL; + char *ovpage = NULL; +#endif #ifdef _WIN32 DWORD len, w2; #else @@ -11157,6 +11171,16 @@ mdb_env_copythr(void *arg) #endif #endif +#if MDB_RPAGE_CACHE + if (my->mc_env->me_encfunc) { + pagebuf = malloc(my->mc_env->me_psize); + if (!pagebuf) { + my->mc_error = ENOMEM; + goto skip; + } + } +#endif + pthread_mutex_lock(&my->mc_mutex); for(;;) { while (!my->mc_new) @@ -11165,6 +11189,36 @@ mdb_env_copythr(void *arg) break; wsize = my->mc_wlen[toggle]; ptr = my->mc_wbuf[toggle]; +#if MDB_RPAGE_CACHE + if (my->mc_env->me_sumfunc || my->mc_env->me_encfunc) { + char *mptr; + for (mptr = ptr; mptr < ptr+wsize; mptr += my->mc_env->me_psize) { + if (((MDB_page *)mptr)->mp_pgno < NUM_METAS) + continue; + if (my->mc_env->me_sumfunc) + mdb_page_set_checksum(my->mc_env, (MDB_page *)mptr, my->mc_env->me_psize); + if (my->mc_env->me_encfunc) { + memcpy(pagebuf, mptr, my->mc_env->me_psize); + mdb_page_encrypt(my->mc_env, (MDB_page *)pagebuf, (MDB_page *)mptr, my->mc_env->me_psize); + } + } + if (my->mc_olen[toggle]) { + if (my->mc_env->me_sumfunc) + mdb_page_set_checksum(my->mc_env, (MDB_page *)my->mc_over[toggle], my->mc_olen[toggle]); + if (my->mc_env->me_encfunc) { + ovpage = malloc(my->mc_olen[toggle]); + if (!ovpage) { + rc = my->mc_error = ENOMEM; + goto skip; + } + mdb_page_encrypt(my->mc_env, (MDB_page *)my->mc_over[toggle], (MDB_page *)ovpage, + my->mc_olen[toggle]); + free(my->mc_over[toggle]); + my->mc_over[toggle] = ovpage; + } + } + } +#endif again: rc = MDB_SUCCESS; while (wsize > 0 && !my->mc_error) { @@ -11193,6 +11247,13 @@ again: my->mc_olen[toggle] = 0; goto again; } +#if MDB_RPAGE_CACHE +skip: + if (ovpage) { + free(ovpage); + ovpage = NULL; + } +#endif my->mc_wlen[toggle] = 0; toggle ^= 1; /* Return the empty buffer to provider */ @@ -11200,6 +11261,10 @@ again: pthread_cond_signal(&my->mc_cond); } pthread_mutex_unlock(&my->mc_mutex); +#if MDB_RPAGE_CACHE + if (pagebuf) + free(pagebuf); +#endif return (THREAD_RET)0; } @@ -11261,6 +11326,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) for (i=0; imc_env->me_psize); + MDB_PAGE_UNREF(my->mc_txn, mc.mc_pg[i]); mc.mc_pg[i] = (MDB_page *)ptr; ptr += my->mc_env->me_psize; } @@ -11286,6 +11352,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) if (mp != leaf) { mc.mc_pg[mc.mc_top] = leaf; mdb_page_copy(leaf, mp, my->mc_env->me_psize); + MDB_PAGE_UNREF(my->mc_txn, mp); mp = leaf; ni = NODEPTR(mp, i); } @@ -11300,22 +11367,46 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) goto done; toggle = my->mc_toggle; } - mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); - memcpy(mo, omp, my->mc_env->me_psize); ovp.op_pgno = my->mc_next_pgno; - mo->mp_pgno = ovp.op_pgno; - mo->mp_txnid = 1; ovp.op_txnid = 1; memcpy(NODEDATA(ni), &ovp, sizeof(ovp)); my->mc_next_pgno += ovp.op_pages; - my->mc_wlen[toggle] += my->mc_env->me_psize; - if (ovp.op_pages > 1) { - my->mc_olen[toggle] = my->mc_env->me_psize * (ovp.op_pages - 1); - my->mc_over[toggle] = (char *)omp + my->mc_env->me_psize; + /* if it's a large overflow page, we want to keep it contiguous + * if we need to do checksum or encryption on it. Otherwise we + * can just modify the leading page and copy the tail separately. + */ + if ((MDB_REMAPPING(my->mc_env->me_flags) || my->mc_env->me_sumfunc) && ovp.op_pages > 1) { + size_t ovlen = ovp.op_pages * my->mc_env->me_psize; + mo = malloc(ovlen); + if (!mo) { + rc = ENOMEM; + goto done; + } + mdb_page_copy(mo, omp, ovlen); + mo->mp_pgno = ovp.op_pgno; + mo->mp_txnid = 1; + my->mc_over[toggle] = (char *)mo; + my->mc_olen[toggle] = ovlen; + MDB_PAGE_UNREF(my->mc_txn, omp); rc = mdb_env_cthr_toggle(my, 1); if (rc) goto done; toggle = my->mc_toggle; + } else { + mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); + memcpy(mo, omp, my->mc_env->me_psize); + mo->mp_pgno = ovp.op_pgno; + mo->mp_txnid = 1; + my->mc_wlen[toggle] += my->mc_env->me_psize; + MDB_PAGE_UNREF(my->mc_txn, omp); + if (ovp.op_pages > 1) { + my->mc_olen[toggle] = my->mc_env->me_psize * (ovp.op_pages - 1); + my->mc_over[toggle] = (char *)omp + my->mc_env->me_psize; + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } } } else if (ni->mn_flags & F_SUBDATA) { MDB_db db; @@ -11324,6 +11415,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) if (mp != leaf) { mc.mc_pg[mc.mc_top] = leaf; mdb_page_copy(leaf, mp, my->mc_env->me_psize); + MDB_PAGE_UNREF(my->mc_txn, mp); mp = leaf; ni = NODEPTR(mp, i); } @@ -11356,6 +11448,7 @@ again: * we must proceed all the way down to its first leaf. */ mdb_page_copy(mc.mc_pg[mc.mc_top], mp, my->mc_env->me_psize); + MDB_PAGE_UNREF(my->mc_txn, mp); goto again; } else mc.mc_pg[mc.mc_top] = mp; @@ -11370,6 +11463,7 @@ again: } mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); mdb_page_copy(mo, mp, my->mc_env->me_psize); + MDB_PAGE_UNREF(my->mc_txn, mp); mo->mp_pgno = my->mc_next_pgno++; mo->mp_txnid = 1; my->mc_wlen[toggle] += my->mc_env->me_psize; @@ -11517,6 +11611,13 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) #else ssize_t len; size_t w2; +#endif +#if MDB_RPAGE_CACHE +#ifdef _WIN32 + LARGE_INTEGER off = 0; +#else + off_t off = 0; +#endif #endif /* Do the lock/unlock of the reader mutex before starting the @@ -11575,10 +11676,25 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) if (w3 > fsize) w3 = fsize; } +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { + off = wsize; + } +#endif wsize = w3 - wsize; while (wsize > 0) { w2 = (wsize > MAX_WRITE) ? MAX_WRITE : wsize; +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { + MAP(rc, env, ptr, w2, off); + } +#endif DO_WRITE(rc, fd, ptr, w2, len); +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { + munmap(ptr, w2); + } +#endif if (!rc) { rc = ErrCode(); break; @@ -11586,6 +11702,11 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) rc = MDB_SUCCESS; ptr += len; wsize -= len; +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { + off += len; + } +#endif continue; } else { rc = MDB_SHORT_WRITE; @@ -11653,11 +11774,16 @@ mdb_env_copy(MDB_env *env, const char *path) int ESECT mdb_env_incr_dumpfd(MDB_env *env, HANDLE fd, size_t txnid) { - int rc; - MDB_page *mp, *mend; + int rc, npgs; + MDB_page *mp; +#if MDB_RPAGE_CACHE + MDB_page *m2; +#endif MDB_txn *txn; mdb_size_t wsize; char *buf = NULL; + pgno_t pg; + MDB_cursor mc; #ifdef _WIN32 DWORD len, w2; #else @@ -11723,15 +11849,33 @@ mdb_env_incr_dumpfd(MDB_env *env, HANDLE fd, size_t txnid) ALIGNED_FREE(buf); buf = NULL; - mp = (MDB_page *)((char *)env->me_map + 2*env->me_psize); - mend = (MDB_page *)((char *)env->me_map + txn->mt_next_pgno * env->me_psize); - while (mp < mend) { + /* dummy cursor for mdb_page_get() */ + mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); + pg = 2; + while (pg < txn->mt_next_pgno) { + if ((rc = MDB_PAGE_GET(&mc, pg, 1, &mp)) != 0) + goto leave; wsize = env->me_psize; - if (IS_OVERFLOW(mp)) + npgs = 1; + if (IS_OVERFLOW(mp)) { + npgs = mp->mp_pages; wsize *= mp->mp_pages; + } +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { + mdb_page_unref(txn, mp); + rc = mdb_rpage_get(txn, pg, npgs, &mp, &m2); + if (rc) + goto leave; + } +#endif if (mp->mp_txnid > txnid) { mdb_size_t w3 = wsize; char *ptr = (char *)mp; +#if MDB_RPAGE_CACHE + if (env->me_encfunc) + ptr = (char *)m2; +#endif while (w3 > 0) { w2 = (w3 > MAX_WRITE) ? MAX_WRITE : w3; DO_WRITE(rc, fd, ptr, w2, len); @@ -11749,7 +11893,8 @@ mdb_env_incr_dumpfd(MDB_env *env, HANDLE fd, size_t txnid) } } } - mp = (MDB_page *)((char *)mp + wsize); + MDB_PAGE_UNREF(txn, mp); + pg += npgs; } leave: @@ -11783,20 +11928,32 @@ int ESECT mdb_env_incr_loadfd(MDB_env *env, HANDLE fd) { #ifdef _WIN32 - DWORD rsize, rlen; + DWORD rsize, rlen, w2; + LARGE_INTEGER off; #else - size_t rsize; + size_t rsize, w2; ssize_t rlen; + off_t off; #endif - char buf[PAGEHDRSZ], *ptr; - MDB_page *rp = (MDB_page *)buf, *mp; + char buf[PAGEHDRSZ], *ptr, *pbuf; + MDB_page *rp; + pgno_t pg, prevpg = 0; + int numpgs, numprev = 0; + int rc = 0; - if (!(env->me_flags & MDB_WRITEMAP)) - return EINVAL; + pbuf = malloc(env->me_psize); + if (!pbuf) + return ENOMEM; + +#ifdef _WIN32 + SetFilePointerEx(env->me_fd, 0, NULL, FILE_BEGIN); +#else + lseek(env->me_fd, 0, SEEK_SET); +#endif for (;;) { #ifdef _WIN32 - int rc = ReadFile(fd, buf, sizeof(buf), &rlen, NULL) ? (int)rlen : -1; + rc = ReadFile(fd, buf, sizeof(buf), &rlen, NULL) ? (int)rlen : -1; if (rc == -1 && ErrCode() == ERROR_HANDLE_EOF) rc = 0; #else @@ -11805,12 +11962,42 @@ mdb_env_incr_loadfd(MDB_env *env, HANDLE fd) if (rlen != sizeof(buf)) break; rsize = env->me_psize; - if (IS_OVERFLOW(rp)) + rp = (MDB_page *)buf; + pg = rp->mp_pgno; +#if MDB_RPAGE_CACHE + /* decrypt the header if needed */ + if (env->me_encfunc && pg > 1) { + MDB_val in, out, enckeys[3]; + int xsize = sizeof(pgno_t) + sizeof(txnid_t); + in.mv_size = PAGEHDRSZ - xsize; + in.mv_data = buf + xsize; + out.mv_size = in.mv_size; + out.mv_data = pbuf + xsize; + enckeys[0] = env->me_enckey; + enckeys[1].mv_size = xsize; + enckeys[1].mv_data = buf; + enckeys[2].mv_size = 0; + enckeys[2].mv_data = pbuf + xsize; /* dummy */ + env->me_encfunc(&in, &out, enckeys, 0); + rp = (MDB_page *)pbuf; + } +#endif + numpgs = 1; + if (IS_OVERFLOW(rp)) { + numpgs = rp->mp_pages; rsize *= rp->mp_pages; + if (rp->mp_pages > 1) { + ptr = realloc(pbuf, rsize); + if (!ptr) { + free(pbuf); + return ENOMEM; + } + pbuf = ptr; + } + } + memcpy(pbuf, buf, sizeof(buf)); rsize -= rlen; - mp = (MDB_page *)(env->me_map + rp->mp_pgno * env->me_psize); - ptr = METADATA(mp); - memcpy(mp, rp, sizeof(buf)); + ptr = pbuf + sizeof(buf); while (rsize > 0) { #ifdef _WIN32 rc = ReadFile(fd, ptr, rsize, &rlen, NULL) ? (int)rlen : -1; @@ -11824,8 +12011,39 @@ mdb_env_incr_loadfd(MDB_env *env, HANDLE fd) ptr += rlen; rsize -= rlen; } + off = (pg-prevpg-numprev) * env->me_psize; + rsize = numpgs * env->me_psize; + if (off) { +#ifdef _WIN32 + SetFilePointerEx(env->me_fd, off, NULL, FILE_CURRENT); +#else + lseek(env->me_fd, off, SEEK_CUR); +#endif + } + ptr = pbuf; + while (rsize > 0) { + w2 = (rsize > MAX_WRITE) ? MAX_WRITE: rsize; + DO_WRITE(rc, env->me_fd, ptr, w2, rlen); + if (!rc) { + rc = ErrCode(); + break; + } else if (rlen > 0) { + rc = MDB_SUCCESS; + ptr += rlen; + rsize -= rlen; + continue; + } else { + rc = MDB_SHORT_WRITE; + break; + } + } + if (rc) + break; + numprev = numpgs; + prevpg = pg; } - return MDB_SUCCESS; + free(pbuf); + return rc; } int ESECT diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1 index 858db1d47e..dc56784cfb 100644 --- a/libraries/liblmdb/mdb_dump.1 +++ b/libraries/liblmdb/mdb_dump.1 @@ -102,6 +102,12 @@ is invoked without including the option when reloading those records, the new databases will likely be damaged beyond repair, permitting neither record storage nor retrieval. +Incremental backup on encrypted databases requires the encryption mechanism +to allow partial decryption of a page. This is because decryption of the +page header is required in order to check if a page is an overflow page. +If the encryption mechanism fails when decrypting only a page header, +then the backup is unusable and cannot be restored. + .SH "SEE ALSO" .BR mdb_load (1) .SH AUTHOR diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c index a31797f5d8..5a6734ba60 100644 --- a/libraries/liblmdb/mdb_dump.c +++ b/libraries/liblmdb/mdb_dump.c @@ -154,6 +154,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name) printf("DATA=END\n"); if (rc == MDB_NOTFOUND) rc = MDB_SUCCESS; + mdb_cursor_close(mc); return rc; } diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1 index 367b36d89a..274ed66137 100644 --- a/libraries/liblmdb/mdb_load.1 +++ b/libraries/liblmdb/mdb_load.1 @@ -60,7 +60,8 @@ on a database that uses custom compare functions. Read from the specified file instead of from the standard input. .TP .BR \-i -Load an incremental backup. +Load an incremental backup. The target environment must not be used +by anything else while this runs. .TP .BR \-n Load an LMDB environment which does not use subdirectories. diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c index 838cff36c3..c58f8e47b8 100644 --- a/libraries/liblmdb/mdb_load.c +++ b/libraries/liblmdb/mdb_load.c @@ -361,6 +361,7 @@ int main(int argc, char *argv[]) break; case 'i': incr = 1; + mode |= NOHDR; break; case 'n': envflags |= MDB_NOSUBDIR; @@ -417,7 +418,6 @@ int main(int argc, char *argv[]) } if (incr) { - envflags |= MDB_WRITEMAP; rc = mdb_env_open(env, envname, envflags, 0664); if (rc) goto openfail; diff --git a/libraries/liblmdb/mtest_enc3.c b/libraries/liblmdb/mtest_enc3.c new file mode 100644 index 0000000000..f73ce28495 --- /dev/null +++ b/libraries/liblmdb/mtest_enc3.c @@ -0,0 +1,225 @@ +/* mtest_enc.c - memory-mapped database tester/toy with encryption */ +/* + * Copyright 2011-2021 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the Symas + * Dual-Use License. + * + * A copy of this license is available in the file LICENSE in the + * source distribution. + */ +#include +#include +#include +#include "lmdb.h" +#include "chacha8.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +// We need bigger entries for the reproducer +#define KEY_SIZE 24 /* ITS#9920 Broken when >= 19 */ +#define MAX_VALUE_SIZE 150 + +static int encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec) +{ + chacha8(src->mv_data, src->mv_size, key[0].mv_data, key[1].mv_data, dst->mv_data); + return 0; +} + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor, *cur2; + MDB_cursor_op op; + MDB_val enckey; + int count; + int *key_lengths, *value_lengths; + char *keys, *values; + char sval[32] = ""; + char ekey[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32}; + + /* srand(time(NULL)); */ + srand(42); + + count = 64000; + key_lengths = (int *)malloc(count*sizeof(int)); + value_lengths = (int *)malloc(count*sizeof(int)); + keys = (char *)malloc(count*KEY_SIZE*sizeof(char)); + values = (char *)malloc(count*MAX_VALUE_SIZE*sizeof(char)); + + // for(i = 0;i= 8 */ + E(mdb_env_open(env, "./testdb", 0 /*|MDB_NOSYNC*/, 0664)); + + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_dbi_open(txn, NULL, 0, &dbi)); + + printf("Adding %d values\n", count); + for (i=0;i in each iteration, since MDB_NOOVERWRITE may modify it */ + data.mv_size = value_lengths[i]; + data.mv_data = &values[i*MAX_VALUE_SIZE]; + if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) { + j++; + data.mv_size = value_lengths[i]; + data.mv_data = &values[i*MAX_VALUE_SIZE]; + } + } + if (j) printf("%d duplicates skipped\n", j); + E(mdb_txn_commit(txn)); + E(mdb_env_stat(env, &mst)); + + E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + + j=0; + key.mv_data = sval; + for (i= count - 1; i > -1; i-= (rand()%5)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + // sprintf(sval, "%03x ", values[i]); + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(key_lengths); + free(value_lengths); + free(keys); + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + printf("Env stat:\n"); + printf("\t ms_entries: %zu\n", mst.ms_entries); + printf("\t ms_depth: %u\n", mst.ms_depth); + E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn)); + + E(mdb_stat(txn, dbi, &mst)); + printf("Database stat:\n"); + printf("\t ms_entries: %zu\n", mst.ms_entries); + printf("\t ms_depth: %u\n", mst.ms_depth); + + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor last\n"); + E(mdb_cursor_get(cursor, &key, &data, MDB_LAST)); + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor last/prev\n"); + E(mdb_cursor_get(cursor, &key, &data, MDB_LAST)); + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + E(mdb_cursor_get(cursor, &key, &data, MDB_PREV)); + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + + printf("Deleting with cursor\n"); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_cursor_open(txn, dbi, &cur2)); + for (i=0; i<50; i++) { + if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, MDB_NEXT))) + break; + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + E(mdb_del(txn, dbi, &key, NULL)); + } + + printf("Restarting cursor in txn\n"); + for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) { + if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, op))) + break; + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cur2); + E(mdb_txn_commit(txn)); + + printf("Restarting cursor outside txn\n"); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) { + if (RES(MDB_NOTFOUND, mdb_cursor_get(cursor, &key, &data, op))) + break; + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + + mdb_dbi_close(env, dbi); + mdb_env_close(env); + + return 0; +} diff --git a/libraries/liblmdb/mtest_enc4.c b/libraries/liblmdb/mtest_enc4.c new file mode 100644 index 0000000000..d72363323c --- /dev/null +++ b/libraries/liblmdb/mtest_enc4.c @@ -0,0 +1,95 @@ +/* mtest_enc.c - memory-mapped database tester/toy with encryption */ +/* + * Copyright 2011-2021 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the Symas + * Dual-Use License. + * + * A copy of this license is available in the file LICENSE in the + * source distribution. + */ +#include +#include +#include +#include +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +MDB_crypto_funcs *cf; + +#define MAX_VALUE_SIZE 65536 +char valbuf[MAX_VALUE_SIZE]; + +/* For ITS#10520: this program creates a DB with records of various sizes, + * to exercise handling of encrypted overflow pages. After creation, the + * various dump/load/copy tools should be run to check for correct operation. + */ +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + int count; + char sval[32] = ""; + char password[] = "This is my passphrase for now..."; + void *mlm; + char *errmsg; + MDB_crypto_funcs *mcf; + + srand(time(NULL)); + + count = 2400; + + E(mdb_env_create(&env)); + mlm = mdb_modload("./crypto.lm", NULL, &mcf, &errmsg); + if (!mlm) { + fprintf(stderr,"Failed to load crypto module: %s\n", errmsg); + exit(1); + } + mdb_modsetup(env, mcf, password); + E(mdb_env_set_maxreaders(env, 1)); + E(mdb_env_set_mapsize(env, 1073741824)); /* 1GB */ + E(mdb_env_open(env, "./testdb", 0 /*|MDB_NOSYNC*/, 0664)); + + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_dbi_open(txn, NULL, 0, &dbi)); + + key.mv_size = 8; + key.mv_data = sval; + + printf("Adding %d values\n", count); + for (i=0;i in each iteration, since MDB_NOOVERWRITE may modify it */ + data.mv_size = rand() % MAX_VALUE_SIZE; + data.mv_data = valbuf; + sprintf(valbuf, "%d foo bar blah", (int)data.mv_size); + if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) { + j++; + data.mv_size = sizeof(sval); + data.mv_data = sval; + } + if (!(i % 100)) { + E(mdb_txn_commit(txn)); + E(mdb_txn_begin(env, NULL, 0, &txn)); + } + } + if (j) printf("%d duplicates skipped\n", j); + E(mdb_txn_commit(txn)); + E(mdb_env_stat(env, &mst)); + + mdb_dbi_close(env, dbi); + mdb_env_close(env); + mdb_modunload(mlm); + + return 0; +}