upstream/openldap
1
0
Fork 0
mirror of https://git.openldap.org/openldap/openldap.git synced 2026-07-15 13:06:52 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/mdb.RE/1.0' into OPENLDAP_REL_ENG_2_7

Browse source
This commit is contained in:
Quanah Gibson-Mount 2026-06-15 16:34:39 +00:00
commit dc2d397b8b
9 changed files with 598 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
libraries/liblmdb/Makefile
View file

@ -112,7 +112,11 @@ mtest6: mtest6.o liblmdb.a
mtest_remap: mtest_remap.o liblmdb.a
mtest_enc: mtest_enc.o chacha8.o liblmdb.a
mtest_enc2: mtest_enc2.o liblmdb.a crypto.lm
$(CC) $(LDFLAGS) -pthread -o $@ mtest_enc2.o liblmdb.a $(LDL)
$(CC) $(LDFLAGS) -pthread -o $@ $@.o liblmdb.a $(LDL)
mtest_enc3: mtest_enc3.o liblmdb.a crypto.lm
$(CC) $(LDFLAGS) -pthread -o $@ $@.o liblmdb.a $(LDL)
mtest_enc4: mtest_enc4.o liblmdb.a crypto.lm
$(CC) $(LDFLAGS) -pthread -o $@ $@.o liblmdb.a $(LDL)
mplay: mplay.o liblmdb.a

10
libraries/liblmdb/crypto.c
View file

@ -57,7 +57,15 @@ static int mcf_encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int
if (encdec) {
rc = ENC(dst, src, key[2].mv_data, key[0].mv_data, iv);
} else {
rc = DEC(dst, src, key[2].mv_data, key[0].mv_data, iv);
if (key[2].mv_size)
rc = DEC(dst, src, key[2].mv_data, key[0].mv_data, iv);
else {
/* Decrypting without MAC check: only used when decrypting page
* headers during incremental load.
*/
rc = crypto_stream_chacha20_ietf_xor_ic(dst->mv_data, src->mv_data,
src->mv_size, iv, 1, key[0].mv_data);
}
}
return rc;
}

290
libraries/liblmdb/mdb.c
View file

@ -1655,7 +1655,7 @@ struct MDB_env {
HANDLE me_lfd; /**< The lock file */
HANDLE me_mfd; /**< For writing and syncing the meta pages */
#ifdef _WIN32
#ifdef MDB_RPAGE_CACHE
#if MDB_RPAGE_CACHE
HANDLE me_fmh; /**< File Mapping handle */
#endif
HANDLE me_ovfd; /**< Overlapped/async with write-through file handle */
@ -5527,7 +5527,7 @@ mdb_env_open2(MDB_env *env, int prev)
env->me_psize = meta.mm_psize;
}
env->me_pagespace = env->me_psize - PAGEHDRSZ
#ifdef MDB_RPAGE_CACHE
#if MDB_RPAGE_CACHE
- env->me_sumsize - env->me_esumsize
#endif
;
@ -6439,8 +6439,10 @@ mdb_env_close_active(MDB_env *env, int excl)
if (env->me_rpages) {
MDB_ID3L el = env->me_rpages;
unsigned int x;
for (x=1; x<=el[0].mid; x++)
for (x=1; x<=el[0].mid; x++) {
munmap(el[x].mptr, el[x].mcnt * env->me_psize);
free(el[x].menc);
}
free(el);
}
}
@ -6905,10 +6907,11 @@ mdb_rpage_encsum(MDB_env *env, MDB_ID3 *id3, unsigned rem, int numpgs)
* @param[in] pg0 the page number for the page to retrieve.
* @param[in] numpgs number of database pages (can be > 1 for overflow pages)
* @param[out] ret address of a pointer where the page's address will be stored.
* @param[out] enc address of a pointer where the encrypted page's address will be stored.
* @return 0 on success, non-zero on failure.
*/
static int
mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret, MDB_page **enc)
{
MDB_env *env = txn->mt_env;
MDB_page *p;
@ -7110,8 +7113,10 @@ retry:
id3.mid = pg0;
if (env->me_encfunc || env->me_sumfunc) {
rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
if (rc)
if (rc) {
free(id3.menc);
goto fail;
}
el[x].muse = id3.muse;
}
pthread_mutex_unlock(&env->me_rpmutex);
@ -7178,8 +7183,10 @@ fail:
}
if (env->me_encfunc || env->me_sumfunc) {
rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
if (rc)
if (rc) {
free(id3.menc);
goto fail;
}
}
mdb_mid3l_insert(el, &id3);
pthread_mutex_unlock(&env->me_rpmutex);
@ -7205,6 +7212,9 @@ ok:
}
#endif
*ret = p;
if (enc && env->me_encfunc) {
*enc = (MDB_page *)(id3.mptr + rem * env->me_psize);
}
return rc;
}
@ -7388,7 +7398,7 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno,
mapped:
#if MDB_RPAGE_CACHE
if (MDB_REMAPPING(txn->mt_env->me_flags)) {
int rc = mdb_rpage_get(txn, pgno, numpgs, &p);
int rc = mdb_rpage_get(txn, pgno, numpgs, &p, NULL);
if (rc) {
txn->mt_flags |= MDB_TXN_ERROR;
return rc;
@ -8571,10 +8581,10 @@ _mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
return MDB_BAD_VALSIZE;
#if SIZE_MAX > MAXDATASIZE
if (data->mv_size > ((mc->mc_db->md_flags & MDB_DUPSORT) ? ENV_MAXKEY(env) : MAXDATASIZE))
if (data->mv_size-1 >= ((mc->mc_db->md_flags & MDB_DUPSORT) ? ENV_MAXKEY(env) : MAXDATASIZE))
return MDB_BAD_VALSIZE;
#else
if ((mc->mc_db->md_flags & MDB_DUPSORT) && data->mv_size > ENV_MAXKEY(env))
if ((mc->mc_db->md_flags & MDB_DUPSORT) && data->mv_size-1 >= ENV_MAXKEY(env))
return MDB_BAD_VALSIZE;
#endif
@ -11136,6 +11146,10 @@ mdb_env_copythr(void *arg)
char *ptr;
int toggle = 0, rc;
mdb_size_t wsize;
#if MDB_RPAGE_CACHE
char *pagebuf = NULL;
char *ovpage = NULL;
#endif
#ifdef _WIN32
DWORD len, w2;
#else
@ -11157,6 +11171,16 @@ mdb_env_copythr(void *arg)
#endif
#endif
#if MDB_RPAGE_CACHE
if (my->mc_env->me_encfunc) {
pagebuf = malloc(my->mc_env->me_psize);
if (!pagebuf) {
my->mc_error = ENOMEM;
goto skip;
}
}
#endif
pthread_mutex_lock(&my->mc_mutex);
for(;;) {
while (!my->mc_new)
@ -11165,6 +11189,36 @@ mdb_env_copythr(void *arg)
break;
wsize = my->mc_wlen[toggle];
ptr = my->mc_wbuf[toggle];
#if MDB_RPAGE_CACHE
if (my->mc_env->me_sumfunc || my->mc_env->me_encfunc) {
char *mptr;
for (mptr = ptr; mptr < ptr+wsize; mptr += my->mc_env->me_psize) {
if (((MDB_page *)mptr)->mp_pgno < NUM_METAS)
continue;
if (my->mc_env->me_sumfunc)
mdb_page_set_checksum(my->mc_env, (MDB_page *)mptr, my->mc_env->me_psize);
if (my->mc_env->me_encfunc) {
memcpy(pagebuf, mptr, my->mc_env->me_psize);
mdb_page_encrypt(my->mc_env, (MDB_page *)pagebuf, (MDB_page *)mptr, my->mc_env->me_psize);
}
}
if (my->mc_olen[toggle]) {
if (my->mc_env->me_sumfunc)
mdb_page_set_checksum(my->mc_env, (MDB_page *)my->mc_over[toggle], my->mc_olen[toggle]);
if (my->mc_env->me_encfunc) {
ovpage = malloc(my->mc_olen[toggle]);
if (!ovpage) {
rc = my->mc_error = ENOMEM;
goto skip;
}
mdb_page_encrypt(my->mc_env, (MDB_page *)my->mc_over[toggle], (MDB_page *)ovpage,
my->mc_olen[toggle]);
free(my->mc_over[toggle]);
my->mc_over[toggle] = ovpage;
}
}
}
#endif
again:
rc = MDB_SUCCESS;
while (wsize > 0 && !my->mc_error) {
@ -11193,6 +11247,13 @@ again:
my->mc_olen[toggle] = 0;
goto again;
}
#if MDB_RPAGE_CACHE
skip:
if (ovpage) {
free(ovpage);
ovpage = NULL;
}
#endif
my->mc_wlen[toggle] = 0;
toggle ^= 1;
/* Return the empty buffer to provider */
@ -11200,6 +11261,10 @@ again:
pthread_cond_signal(&my->mc_cond);
}
pthread_mutex_unlock(&my->mc_mutex);
#if MDB_RPAGE_CACHE
if (pagebuf)
free(pagebuf);
#endif
return (THREAD_RET)0;
}
@ -11261,6 +11326,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
for (i=0; i<mc.mc_top; i++) {
mdb_page_copy((MDB_page *)ptr, mc.mc_pg[i], my->mc_env->me_psize);
MDB_PAGE_UNREF(my->mc_txn, mc.mc_pg[i]);
mc.mc_pg[i] = (MDB_page *)ptr;
ptr += my->mc_env->me_psize;
}
@ -11286,6 +11352,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
if (mp != leaf) {
mc.mc_pg[mc.mc_top] = leaf;
mdb_page_copy(leaf, mp, my->mc_env->me_psize);
MDB_PAGE_UNREF(my->mc_txn, mp);
mp = leaf;
ni = NODEPTR(mp, i);
}
@ -11300,22 +11367,46 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
goto done;
toggle = my->mc_toggle;
}
mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]);
memcpy(mo, omp, my->mc_env->me_psize);
ovp.op_pgno = my->mc_next_pgno;
mo->mp_pgno = ovp.op_pgno;
mo->mp_txnid = 1;
ovp.op_txnid = 1;
memcpy(NODEDATA(ni), &ovp, sizeof(ovp));
my->mc_next_pgno += ovp.op_pages;
my->mc_wlen[toggle] += my->mc_env->me_psize;
if (ovp.op_pages > 1) {
my->mc_olen[toggle] = my->mc_env->me_psize * (ovp.op_pages - 1);
my->mc_over[toggle] = (char *)omp + my->mc_env->me_psize;
/* if it's a large overflow page, we want to keep it contiguous
* if we need to do checksum or encryption on it. Otherwise we
* can just modify the leading page and copy the tail separately.
*/
if ((MDB_REMAPPING(my->mc_env->me_flags) || my->mc_env->me_sumfunc) && ovp.op_pages > 1) {
size_t ovlen = ovp.op_pages * my->mc_env->me_psize;
mo = malloc(ovlen);
if (!mo) {
rc = ENOMEM;
goto done;
}
mdb_page_copy(mo, omp, ovlen);
mo->mp_pgno = ovp.op_pgno;
mo->mp_txnid = 1;
my->mc_over[toggle] = (char *)mo;
my->mc_olen[toggle] = ovlen;
MDB_PAGE_UNREF(my->mc_txn, omp);
rc = mdb_env_cthr_toggle(my, 1);
if (rc)
goto done;
toggle = my->mc_toggle;
} else {
mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]);
memcpy(mo, omp, my->mc_env->me_psize);
mo->mp_pgno = ovp.op_pgno;
mo->mp_txnid = 1;
my->mc_wlen[toggle] += my->mc_env->me_psize;
MDB_PAGE_UNREF(my->mc_txn, omp);
if (ovp.op_pages > 1) {
my->mc_olen[toggle] = my->mc_env->me_psize * (ovp.op_pages - 1);
my->mc_over[toggle] = (char *)omp + my->mc_env->me_psize;
rc = mdb_env_cthr_toggle(my, 1);
if (rc)
goto done;
toggle = my->mc_toggle;
}
}
} else if (ni->mn_flags & F_SUBDATA) {
MDB_db db;
@ -11324,6 +11415,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
if (mp != leaf) {
mc.mc_pg[mc.mc_top] = leaf;
mdb_page_copy(leaf, mp, my->mc_env->me_psize);
MDB_PAGE_UNREF(my->mc_txn, mp);
mp = leaf;
ni = NODEPTR(mp, i);
}
@ -11356,6 +11448,7 @@ again:
* we must proceed all the way down to its first leaf.
*/
mdb_page_copy(mc.mc_pg[mc.mc_top], mp, my->mc_env->me_psize);
MDB_PAGE_UNREF(my->mc_txn, mp);
goto again;
} else
mc.mc_pg[mc.mc_top] = mp;
@ -11370,6 +11463,7 @@ again:
}
mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]);
mdb_page_copy(mo, mp, my->mc_env->me_psize);
MDB_PAGE_UNREF(my->mc_txn, mp);
mo->mp_pgno = my->mc_next_pgno++;
mo->mp_txnid = 1;
my->mc_wlen[toggle] += my->mc_env->me_psize;
@ -11517,6 +11611,13 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
#else
ssize_t len;
size_t w2;
#endif
#if MDB_RPAGE_CACHE
#ifdef _WIN32
LARGE_INTEGER off = 0;
#else
off_t off = 0;
#endif
#endif
/* Do the lock/unlock of the reader mutex before starting the
@ -11575,10 +11676,25 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
if (w3 > fsize)
w3 = fsize;
}
#if MDB_RPAGE_CACHE
if (MDB_REMAPPING(env->me_flags)) {
off = wsize;
}
#endif
wsize = w3 - wsize;
while (wsize > 0) {
w2 = (wsize > MAX_WRITE) ? MAX_WRITE : wsize;
#if MDB_RPAGE_CACHE
if (MDB_REMAPPING(env->me_flags)) {
MAP(rc, env, ptr, w2, off);
}
#endif
DO_WRITE(rc, fd, ptr, w2, len);
#if MDB_RPAGE_CACHE
if (MDB_REMAPPING(env->me_flags)) {
munmap(ptr, w2);
}
#endif
if (!rc) {
rc = ErrCode();
break;
@ -11586,6 +11702,11 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
rc = MDB_SUCCESS;
ptr += len;
wsize -= len;
#if MDB_RPAGE_CACHE
if (MDB_REMAPPING(env->me_flags)) {
off += len;
}
#endif
continue;
} else {
rc = MDB_SHORT_WRITE;
@ -11653,11 +11774,16 @@ mdb_env_copy(MDB_env *env, const char *path)
int ESECT
mdb_env_incr_dumpfd(MDB_env *env, HANDLE fd, size_t txnid)
{
int rc;
MDB_page *mp, *mend;
int rc, npgs;
MDB_page *mp;
#if MDB_RPAGE_CACHE
MDB_page *m2;
#endif
MDB_txn *txn;
mdb_size_t wsize;
char *buf = NULL;
pgno_t pg;
MDB_cursor mc;
#ifdef _WIN32
DWORD len, w2;
#else
@ -11723,15 +11849,33 @@ mdb_env_incr_dumpfd(MDB_env *env, HANDLE fd, size_t txnid)
ALIGNED_FREE(buf);
buf = NULL;
mp = (MDB_page *)((char *)env->me_map + 2*env->me_psize);
mend = (MDB_page *)((char *)env->me_map + txn->mt_next_pgno * env->me_psize);
while (mp < mend) {
/* dummy cursor for mdb_page_get() */
mdb_cursor_init(&mc, txn, MAIN_DBI, NULL);
pg = 2;
while (pg < txn->mt_next_pgno) {
if ((rc = MDB_PAGE_GET(&mc, pg, 1, &mp)) != 0)
goto leave;
wsize = env->me_psize;
if (IS_OVERFLOW(mp))
npgs = 1;
if (IS_OVERFLOW(mp)) {
npgs = mp->mp_pages;
wsize *= mp->mp_pages;
}
#if MDB_RPAGE_CACHE
if (MDB_REMAPPING(env->me_flags)) {
mdb_page_unref(txn, mp);
rc = mdb_rpage_get(txn, pg, npgs, &mp, &m2);
if (rc)
goto leave;
}
#endif
if (mp->mp_txnid > txnid) {
mdb_size_t w3 = wsize;
char *ptr = (char *)mp;
#if MDB_RPAGE_CACHE
if (env->me_encfunc)
ptr = (char *)m2;
#endif
while (w3 > 0) {
w2 = (w3 > MAX_WRITE) ? MAX_WRITE : w3;
DO_WRITE(rc, fd, ptr, w2, len);
@ -11749,7 +11893,8 @@ mdb_env_incr_dumpfd(MDB_env *env, HANDLE fd, size_t txnid)
}
}
}
mp = (MDB_page *)((char *)mp + wsize);
MDB_PAGE_UNREF(txn, mp);
pg += npgs;
}
leave:
@ -11783,20 +11928,32 @@ int ESECT
mdb_env_incr_loadfd(MDB_env *env, HANDLE fd)
{
#ifdef _WIN32
DWORD rsize, rlen;
DWORD rsize, rlen, w2;
LARGE_INTEGER off;
#else
size_t rsize;
size_t rsize, w2;
ssize_t rlen;
off_t off;
#endif
char buf[PAGEHDRSZ], *ptr;
MDB_page *rp = (MDB_page *)buf, *mp;
char buf[PAGEHDRSZ], *ptr, *pbuf;
MDB_page *rp;
pgno_t pg, prevpg = 0;
int numpgs, numprev = 0;
int rc = 0;
if (!(env->me_flags & MDB_WRITEMAP))
return EINVAL;
pbuf = malloc(env->me_psize);
if (!pbuf)
return ENOMEM;
#ifdef _WIN32
SetFilePointerEx(env->me_fd, 0, NULL, FILE_BEGIN);
#else
lseek(env->me_fd, 0, SEEK_SET);
#endif
for (;;) {
#ifdef _WIN32
int rc = ReadFile(fd, buf, sizeof(buf), &rlen, NULL) ? (int)rlen : -1;
rc = ReadFile(fd, buf, sizeof(buf), &rlen, NULL) ? (int)rlen : -1;
if (rc == -1 && ErrCode() == ERROR_HANDLE_EOF)
rc = 0;
#else
@ -11805,12 +11962,42 @@ mdb_env_incr_loadfd(MDB_env *env, HANDLE fd)
if (rlen != sizeof(buf))
break;
rsize = env->me_psize;
if (IS_OVERFLOW(rp))
rp = (MDB_page *)buf;
pg = rp->mp_pgno;
#if MDB_RPAGE_CACHE
/* decrypt the header if needed */
if (env->me_encfunc && pg > 1) {
MDB_val in, out, enckeys[3];
int xsize = sizeof(pgno_t) + sizeof(txnid_t);
in.mv_size = PAGEHDRSZ - xsize;
in.mv_data = buf + xsize;
out.mv_size = in.mv_size;
out.mv_data = pbuf + xsize;
enckeys[0] = env->me_enckey;
enckeys[1].mv_size = xsize;
enckeys[1].mv_data = buf;
enckeys[2].mv_size = 0;
enckeys[2].mv_data = pbuf + xsize; /* dummy */
env->me_encfunc(&in, &out, enckeys, 0);
rp = (MDB_page *)pbuf;
}
#endif
numpgs = 1;
if (IS_OVERFLOW(rp)) {
numpgs = rp->mp_pages;
rsize *= rp->mp_pages;
if (rp->mp_pages > 1) {
ptr = realloc(pbuf, rsize);
if (!ptr) {
free(pbuf);
return ENOMEM;
}
pbuf = ptr;
}
}
memcpy(pbuf, buf, sizeof(buf));
rsize -= rlen;
mp = (MDB_page *)(env->me_map + rp->mp_pgno * env->me_psize);
ptr = METADATA(mp);
memcpy(mp, rp, sizeof(buf));
ptr = pbuf + sizeof(buf);
while (rsize > 0) {
#ifdef _WIN32
rc = ReadFile(fd, ptr, rsize, &rlen, NULL) ? (int)rlen : -1;
@ -11824,8 +12011,39 @@ mdb_env_incr_loadfd(MDB_env *env, HANDLE fd)
ptr += rlen;
rsize -= rlen;
}
off = (pg-prevpg-numprev) * env->me_psize;
rsize = numpgs * env->me_psize;
if (off) {
#ifdef _WIN32
SetFilePointerEx(env->me_fd, off, NULL, FILE_CURRENT);
#else
lseek(env->me_fd, off, SEEK_CUR);
#endif
}
ptr = pbuf;
while (rsize > 0) {
w2 = (rsize > MAX_WRITE) ? MAX_WRITE: rsize;
DO_WRITE(rc, env->me_fd, ptr, w2, rlen);
if (!rc) {
rc = ErrCode();
break;
} else if (rlen > 0) {
rc = MDB_SUCCESS;
ptr += rlen;
rsize -= rlen;
continue;
} else {
rc = MDB_SHORT_WRITE;
break;
}
}
if (rc)
break;
numprev = numpgs;
prevpg = pg;
}
return MDB_SUCCESS;
free(pbuf);
return rc;
}
int ESECT

6
libraries/liblmdb/mdb_dump.1
View file

@ -102,6 +102,12 @@ is invoked without including the
option when reloading those records, the new databases will likely be
damaged beyond repair, permitting neither record storage nor retrieval.
Incremental backup on encrypted databases requires the encryption mechanism
to allow partial decryption of a page. This is because decryption of the
page header is required in order to check if a page is an overflow page.
If the encryption mechanism fails when decrypting only a page header,
then the backup is unusable and cannot be restored.
.SH "SEE ALSO"
.BR mdb_load (1)
.SH AUTHOR

1
libraries/liblmdb/mdb_dump.c
View file

@ -154,6 +154,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
printf("DATA=END\n");
if (rc == MDB_NOTFOUND)
rc = MDB_SUCCESS;
mdb_cursor_close(mc);
return rc;
}

3
libraries/liblmdb/mdb_load.1
View file

@ -60,7 +60,8 @@ on a database that uses custom compare functions.
Read from the specified file instead of from the standard input.
.TP
.BR \-i
Load an incremental backup.
Load an incremental backup. The target environment must not be used
by anything else while this runs.
.TP
.BR \-n
Load an LMDB environment which does not use subdirectories.

2
libraries/liblmdb/mdb_load.c
View file

@ -361,6 +361,7 @@ int main(int argc, char *argv[])
break;
case 'i':
incr = 1;
mode |= NOHDR;
break;
case 'n':
envflags |= MDB_NOSUBDIR;
@ -417,7 +418,6 @@ int main(int argc, char *argv[])
}
if (incr) {
envflags |= MDB_WRITEMAP;
rc = mdb_env_open(env, envname, envflags, 0664);
if (rc)
goto openfail;

225
libraries/liblmdb/mtest_enc3.c Normal file
View file

@ -0,0 +1,225 @@
/* mtest_enc.c - memory-mapped database tester/toy with encryption */
/*
* Copyright 2011-2021 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the Symas
* Dual-Use License.
*
* A copy of this license is available in the file LICENSE in the
* source distribution.
*/
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include "lmdb.h"
#include "chacha8.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
// We need bigger entries for the reproducer
#define KEY_SIZE 24 /* ITS#9920 Broken when >= 19 */
#define MAX_VALUE_SIZE 150
static int encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
{
chacha8(src->mv_data, src->mv_size, key[0].mv_data, key[1].mv_data, dst->mv_data);
return 0;
}
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor, *cur2;
MDB_cursor_op op;
MDB_val enckey;
int count;
int *key_lengths, *value_lengths;
char *keys, *values;
char sval[32] = "";
char ekey[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32};
/* srand(time(NULL)); */
srand(42);
count = 64000;
key_lengths = (int *)malloc(count*sizeof(int));
value_lengths = (int *)malloc(count*sizeof(int));
keys = (char *)malloc(count*KEY_SIZE*sizeof(char));
values = (char *)malloc(count*MAX_VALUE_SIZE*sizeof(char));
// for(i = 0;i<count;i++) {
// values[i] = i;
// }
for(i=0;i<count;i++) {
key_lengths[i] = KEY_SIZE;
value_lengths[i] = rand() % MAX_VALUE_SIZE + 1;
// Generate random key and value data
for(int k=0;k<key_lengths[i];k++) {
keys[i*KEY_SIZE+k] = 'A' + (rand() % 26);
}
for(int k=0;k<value_lengths[i];k++) {
values[i*MAX_VALUE_SIZE+k] = 'A' + (rand() % 26);
}
}
// return 0;
enckey.mv_data = ekey;
enckey.mv_size = sizeof(ekey);
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env, 1));
E(mdb_env_set_mapsize(env, 1073741824 /*1GiB*/));
E(mdb_env_set_encrypt(env, encfunc, &enckey, 16)); /* ITS#9920 Broken when >= 8 */
E(mdb_env_open(env, "./testdb", 0 /*|MDB_NOSYNC*/, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_dbi_open(txn, NULL, 0, &dbi));
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
key.mv_size = key_lengths[i];
key.mv_data = &keys[i*KEY_SIZE];
/* Set <data> in each iteration, since MDB_NOOVERWRITE may modify it */
data.mv_size = value_lengths[i];
data.mv_data = &values[i*MAX_VALUE_SIZE];
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) {
j++;
data.mv_size = value_lengths[i];
data.mv_data = &values[i*MAX_VALUE_SIZE];
}
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
j=0;
key.mv_data = sval;
for (i= count - 1; i > -1; i-= (rand()%5)) {
j++;
txn=NULL;
E(mdb_txn_begin(env, NULL, 0, &txn));
// sprintf(sval, "%03x ", values[i]);
if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
j--;
mdb_txn_abort(txn);
} else {
E(mdb_txn_commit(txn));
}
}
free(key_lengths);
free(value_lengths);
free(keys);
free(values);
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
printf("Env stat:\n");
printf("\t ms_entries: %zu\n", mst.ms_entries);
printf("\t ms_depth: %u\n", mst.ms_depth);
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_stat(txn, dbi, &mst));
printf("Database stat:\n");
printf("\t ms_entries: %zu\n", mst.ms_entries);
printf("\t ms_depth: %u\n", mst.ms_depth);
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor last\n");
E(mdb_cursor_get(cursor, &key, &data, MDB_LAST));
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
printf("Cursor prev\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
printf("Cursor last/prev\n");
E(mdb_cursor_get(cursor, &key, &data, MDB_LAST));
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
E(mdb_cursor_get(cursor, &key, &data, MDB_PREV));
printf("key: %.*s, data: %.*s\n",
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
printf("Deleting with cursor\n");
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_cursor_open(txn, dbi, &cur2));
for (i=0; i<50; i++) {
if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, MDB_NEXT)))
break;
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
E(mdb_del(txn, dbi, &key, NULL));
}
printf("Restarting cursor in txn\n");
for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) {
if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, op)))
break;
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
mdb_cursor_close(cur2);
E(mdb_txn_commit(txn));
printf("Restarting cursor outside txn\n");
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) {
if (RES(MDB_NOTFOUND, mdb_cursor_get(cursor, &key, &data, op)))
break;
printf("key: %p %.*s, data: %p %.*s\n",
key.mv_data, (int) key.mv_size, (char *) key.mv_data,
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

95
libraries/liblmdb/mtest_enc4.c Normal file
View file

@ -0,0 +1,95 @@
/* mtest_enc.c - memory-mapped database tester/toy with encryption */
/*
* Copyright 2011-2021 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the Symas
* Dual-Use License.
*
* A copy of this license is available in the file LICENSE in the
* source distribution.
*/
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <time.h>
#include "lmdb.h"
#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
MDB_crypto_funcs *cf;
#define MAX_VALUE_SIZE 65536
char valbuf[MAX_VALUE_SIZE];
/* For ITS#10520: this program creates a DB with records of various sizes,
* to exercise handling of encrypted overflow pages. After creation, the
* various dump/load/copy tools should be run to check for correct operation.
*/
int main(int argc,char * argv[])
{
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_txn *txn;
MDB_stat mst;
int count;
char sval[32] = "";
char password[] = "This is my passphrase for now...";
void *mlm;
char *errmsg;
MDB_crypto_funcs *mcf;
srand(time(NULL));
count = 2400;
E(mdb_env_create(&env));
mlm = mdb_modload("./crypto.lm", NULL, &mcf, &errmsg);
if (!mlm) {
fprintf(stderr,"Failed to load crypto module: %s\n", errmsg);
exit(1);
}
mdb_modsetup(env, mcf, password);
E(mdb_env_set_maxreaders(env, 1));
E(mdb_env_set_mapsize(env, 1073741824)); /* 1GB */
E(mdb_env_open(env, "./testdb", 0 /*|MDB_NOSYNC*/, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_dbi_open(txn, NULL, 0, &dbi));
key.mv_size = 8;
key.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%08x", rand());
/* Set <data> in each iteration, since MDB_NOOVERWRITE may modify it */
data.mv_size = rand() % MAX_VALUE_SIZE;
data.mv_data = valbuf;
sprintf(valbuf, "%d foo bar blah", (int)data.mv_size);
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) {
j++;
data.mv_size = sizeof(sval);
data.mv_data = sval;
}
if (!(i % 100)) {
E(mdb_txn_commit(txn));
E(mdb_txn_begin(env, NULL, 0, &txn));
}
}
if (j) printf("%d duplicates skipped\n", j);
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
mdb_dbi_close(env, dbi);
mdb_env_close(env);
mdb_modunload(mlm);
return 0;
}