From 2fc9b2a308a7d72edede919956a0d290a72a7781 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Sun, 28 Apr 2019 17:19:07 +0200 Subject: libq/{vdb,cache}: unify sorted and unsorted traversal Get vdb to know some bits for cache, such that cache can be a more shallow wrapper around it. Use the same code to traverse in sorted mode by just keeping a temp list in the sorted case. Signed-off-by: Fabian Groffen --- libq/cache.c | 137 ++++++++++++++++++------------------------------- libq/cache.h | 14 +---- libq/vdb.c | 164 ++++++++++++++++++++++++++++++++++------------------------- libq/vdb.h | 46 +++++++++++++---- qgrep.c | 4 +- 5 files changed, 186 insertions(+), 179 deletions(-) diff --git a/libq/cache.c b/libq/cache.c index 77242871..a00dd6b9 100644 --- a/libq/cache.c +++ b/libq/cache.c @@ -53,46 +53,43 @@ static const char portrepo_name[] = "profiles/repo_name"; cache_ctx * cache_open(const char *sroot, const char *portdir) { - q_vdb_ctx *dir; cache_ctx *ret; char buf[_Q_PATH_MAX]; + char *repo = NULL; size_t repolen = 0; - ret = xzalloc(sizeof(cache_ctx)); - snprintf(buf, sizeof(buf), "%s%s/%s", sroot, portdir, portrepo_name); - if (eat_file(buf, &ret->repo, &repolen)) { - (void)rmspace(ret->repo); + if (eat_file(buf, &repo, &repolen)) { + (void)rmspace(repo); } else { - ret->repo = NULL; /* ignore missing repo file */ + repo = NULL; /* ignore missing repo file */ } snprintf(buf, sizeof(buf), "%s/%s", portdir, portcachedir_md5); - dir = q_vdb_open2(sroot, buf, true); - if (dir != NULL) { - ret->dir_ctx = dir; + ret = q_vdb_open2(sroot, buf, true); + if (ret != NULL) { ret->cachetype = CACHE_METADATA_MD5; + ret->repo = repo; return ret; } snprintf(buf, sizeof(buf), "%s/%s", portdir, portcachedir_pms); - dir = q_vdb_open2(sroot, buf, true); - if (dir != NULL) { - ret->dir_ctx = dir; + ret = q_vdb_open2(sroot, buf, true); + if (ret != NULL) { ret->cachetype = CACHE_METADATA_PMS; + ret->repo = repo; return ret; } - dir = q_vdb_open2(sroot, portdir, true); - if (dir != NULL) { - ret->dir_ctx = dir; + ret = q_vdb_open2(sroot, portdir, true); + if (ret != NULL) { ret->cachetype = CACHE_EBUILD; + ret->repo = repo; return ret; } cache_close(ret); - warnf("could not open repository at %s (under root %s)", - portdir, sroot); + warnf("could not open repository at %s (under root %s)", portdir, sroot); return NULL; } @@ -100,49 +97,41 @@ cache_open(const char *sroot, const char *portdir) void cache_close(cache_ctx *ctx) { - if (ctx->dir_ctx != NULL) - q_vdb_close(ctx->dir_ctx); if (ctx->repo != NULL) free(ctx->repo); - free(ctx); + if (ctx->ebuilddir_ctx != NULL) + free(ctx->ebuilddir_ctx); + q_vdb_close(ctx); } cache_cat_ctx * cache_open_cat(cache_ctx *ctx, const char *name) { - cache_cat_ctx *ret = q_vdb_open_cat(ctx->dir_ctx, name); - if (ret != NULL) - ret->ctx = (q_vdb_ctx *)ctx; - return ret; + return q_vdb_open_cat(ctx, name); } cache_cat_ctx * cache_next_cat(cache_ctx *ctx) { - cache_cat_ctx *ret = q_vdb_next_cat(ctx->dir_ctx); - if (ret != NULL) - ret->ctx = (q_vdb_ctx *)ctx; - return ret; + return q_vdb_next_cat(ctx); } void cache_close_cat(cache_cat_ctx *cat_ctx) { - q_vdb_close_cat(cat_ctx); + return q_vdb_close_cat(cat_ctx); } cache_pkg_ctx * cache_open_pkg(cache_cat_ctx *cat_ctx, const char *name) { - cache_pkg_ctx *ret = q_vdb_open_pkg(cat_ctx, name); - ret->repo = ((cache_ctx *)cat_ctx->ctx)->repo; - return ret; + return q_vdb_open_pkg(cat_ctx, name); } cache_pkg_ctx * cache_next_pkg(cache_cat_ctx *cat_ctx) { - cache_ctx *ctx = (cache_ctx *)(cat_ctx->ctx); + cache_ctx *ctx = (cache_ctx *)cat_ctx->ctx; cache_pkg_ctx *ret = NULL; if (ctx->cachetype == CACHE_EBUILD) { @@ -152,7 +141,10 @@ cache_next_pkg(cache_cat_ctx *cat_ctx) * to CAT/P like in VDB and metadata */ do { if (ctx->ebuilddir_pkg_ctx == NULL) { - q_vdb_ctx *pkgdir = &ctx->ebuilddir_ctx; + q_vdb_ctx *pkgdir = ctx->ebuilddir_ctx; + + if (pkgdir == NULL) + pkgdir = ctx->ebuilddir_ctx = xzalloc(sizeof(q_vdb_ctx)); if ((ctx->ebuilddir_pkg_ctx = q_vdb_next_pkg(cat_ctx)) == NULL) return NULL; @@ -492,14 +484,16 @@ void cache_close_pkg(cache_pkg_ctx *pkg_ctx) { /* avoid free of cache_ctx' repo by q_vdb_close_pkg */ - if (((cache_ctx *)pkg_ctx->cat_ctx->ctx)->repo == pkg_ctx->repo) + if (pkg_ctx->cat_ctx->ctx->repo == pkg_ctx->repo) pkg_ctx->repo = NULL; + q_vdb_close_pkg(pkg_ctx); } -int -cache_foreach_pkg(const char *sroot, const char *portdir, - q_vdb_pkg_cb callback, void *priv, q_vdb_cat_filter filter) +static int +cache_foreach_pkg_int(const char *sroot, const char *portdir, + q_vdb_pkg_cb callback, void *priv, q_vdb_cat_filter filter, + bool sort, void *catsortfunc, void *pkgsortfunc) { cache_ctx *ctx; cache_cat_ctx *cat_ctx; @@ -510,6 +504,12 @@ cache_foreach_pkg(const char *sroot, const char *portdir, if (!ctx) return EXIT_FAILURE; + ctx->do_sort = sort; + if (catsortfunc != NULL) + ctx->catsortfunc = catsortfunc; + if (pkgsortfunc != NULL) + ctx->pkgsortfunc = pkgsortfunc; + ret = 0; while ((cat_ctx = cache_next_cat(ctx))) { if (filter && !filter(cat_ctx, priv)) @@ -518,63 +518,26 @@ cache_foreach_pkg(const char *sroot, const char *portdir, ret |= callback(pkg_ctx, priv); cache_close_pkg(pkg_ctx); } + cache_close_cat(cat_ctx); } + cache_close(ctx); return ret; } +int +cache_foreach_pkg(const char *sroot, const char *portdir, + q_vdb_pkg_cb callback, void *priv, q_vdb_cat_filter filter) +{ + return cache_foreach_pkg_int(sroot, portdir, callback, priv, + filter, false, NULL, NULL); +} + int cache_foreach_pkg_sorted(const char *sroot, const char *portdir, q_vdb_pkg_cb callback, void *priv, void *catsortfunc, void *pkgsortfunc) { - cache_ctx *ctx; - cache_cat_ctx *cat_ctx; - cache_pkg_ctx *pkg_ctx; - int ret = 0; - int c; - int p; - int cat_cnt; - int pkg_cnt; - struct dirent **cat_de; - struct dirent **pkg_de; - - ctx = cache_open(sroot, portdir); - if (!ctx) - return EXIT_FAILURE; - - if (catsortfunc == NULL) - catsortfunc = alphasort; - if (pkgsortfunc == NULL) - pkgsortfunc = alphasort; - - cat_cnt = scandirat(ctx->dir_ctx->vdb_fd, - ".", &cat_de, q_vdb_filter_cat, catsortfunc); - for (c = 0; c < cat_cnt; c++) { - cat_ctx = cache_open_cat(ctx, cat_de[c]->d_name); - if (!cat_ctx) - continue; - - pkg_cnt = scandirat(ctx->dir_ctx->vdb_fd, - cat_de[c]->d_name, &pkg_de, q_vdb_filter_pkg, pkgsortfunc); - for (p = 0; p < pkg_cnt; p++) { - if (pkg_de[p]->d_name[0] == '-') - continue; - - pkg_ctx = cache_open_pkg(cat_ctx, pkg_de[p]->d_name); - if (!pkg_ctx) - continue; - - ret |= callback(pkg_ctx, priv); - - cache_close_pkg(pkg_ctx); - } - scandir_free(pkg_de, pkg_cnt); - - cache_close_cat(cat_ctx); - } - scandir_free(cat_de, cat_cnt); - - cache_close(ctx); - return ret; + return cache_foreach_pkg_int(sroot, portdir, callback, priv, + NULL, true, catsortfunc, pkgsortfunc); } diff --git a/libq/cache.h b/libq/cache.h index 0157824d..f9b1d430 100644 --- a/libq/cache.h +++ b/libq/cache.h @@ -13,19 +13,7 @@ #include "atom.h" #include "vdb.h" -typedef struct cache_ctx { - q_vdb_ctx *dir_ctx; - enum { - CACHE_UNSET = 0, - CACHE_METADATA_MD5, - CACHE_METADATA_PMS, - CACHE_EBUILD, - } cachetype; - q_vdb_pkg_ctx *ebuilddir_pkg_ctx; - q_vdb_cat_ctx *ebuilddir_cat_ctx; - q_vdb_ctx ebuilddir_ctx; - char *repo; -} cache_ctx; +#define cache_ctx q_vdb_ctx #define cache_cat_ctx q_vdb_cat_ctx #define cache_pkg_ctx q_vdb_pkg_ctx diff --git a/libq/vdb.c b/libq/vdb.c index a32ba53e..9c53b5b5 100644 --- a/libq/vdb.c +++ b/libq/vdb.c @@ -45,6 +45,13 @@ q_vdb_open2(const char *sroot, const char *svdb, bool quiet) if (ctx->dir == NULL) goto cv_error; + ctx->do_sort = false; + ctx->cat_de = NULL; + ctx->catsortfunc = alphasort; + ctx->pkgsortfunc = alphasort; + ctx->repo = NULL; + ctx->ebuilddir_ctx = NULL; + ctx->ebuilddir_pkg_ctx = NULL; return ctx; cv_error: @@ -69,6 +76,8 @@ q_vdb_close(q_vdb_ctx *ctx) /* closedir() above does this for us: */ /* close(ctx->vdb_fd); */ close(ctx->portroot_fd); + if (ctx->do_sort) + scandir_free(ctx->cat_de, ctx->cat_cnt); free(ctx); } @@ -136,6 +145,7 @@ q_vdb_open_cat(q_vdb_ctx *ctx, const char *name) cat_ctx->fd = fd; cat_ctx->dir = dir; cat_ctx->ctx = ctx; + cat_ctx->pkg_de = NULL; return cat_ctx; } @@ -143,22 +153,39 @@ q_vdb_cat_ctx * q_vdb_next_cat(q_vdb_ctx *ctx) { /* search for a category directory */ - q_vdb_cat_ctx *cat_ctx; - const struct dirent *de; + q_vdb_cat_ctx *cat_ctx = NULL; - next_cat: - de = readdir(ctx->dir); - if (!de) { - q_vdb_close(ctx); - return NULL; - } + if (ctx->do_sort) { + if (ctx->cat_de == NULL) { + ctx->cat_cnt = scandirat(ctx->vdb_fd, + ".", &ctx->cat_de, q_vdb_filter_cat, ctx->catsortfunc); + ctx->cat_cur = 0; + } + + while (ctx->cat_cur < ctx->cat_cnt) { + cat_ctx = q_vdb_open_cat(ctx, ctx->cat_de[ctx->cat_cur++]->d_name); + if (!cat_ctx) + continue; + break; + } + } else { + /* cheaper "streaming" variant */ + const struct dirent *de; + do { + de = readdir(ctx->dir); + if (!de) + break; - if (q_vdb_filter_cat(de) == 0) - goto next_cat; + if (q_vdb_filter_cat(de) == 0) + continue; + + cat_ctx = q_vdb_open_cat(ctx, de->d_name); + if (!cat_ctx) + continue; - cat_ctx = q_vdb_open_cat(ctx, de->d_name); - if (!cat_ctx) - goto next_cat; + break; + } while (1); + } return cat_ctx; } @@ -169,6 +196,8 @@ q_vdb_close_cat(q_vdb_cat_ctx *cat_ctx) closedir(cat_ctx->dir); /* closedir() above does this for us: */ /* close(ctx->fd); */ + if (cat_ctx->ctx->do_sort) + scandir_free(cat_ctx->pkg_de, cat_ctx->pkg_cnt); free(cat_ctx); } @@ -194,7 +223,7 @@ q_vdb_open_pkg(q_vdb_cat_ctx *cat_ctx, const char *name) q_vdb_pkg_ctx *pkg_ctx = xmalloc(sizeof(*pkg_ctx)); pkg_ctx->name = name; pkg_ctx->slot = NULL; - pkg_ctx->repo = NULL; + pkg_ctx->repo = cat_ctx->ctx->repo; pkg_ctx->fd = -1; pkg_ctx->cat_ctx = cat_ctx; return pkg_ctx; @@ -203,22 +232,40 @@ q_vdb_open_pkg(q_vdb_cat_ctx *cat_ctx, const char *name) q_vdb_pkg_ctx * q_vdb_next_pkg(q_vdb_cat_ctx *cat_ctx) { - q_vdb_pkg_ctx *pkg_ctx; - const struct dirent *de; + q_vdb_pkg_ctx *pkg_ctx = NULL; - next_pkg: - de = readdir(cat_ctx->dir); - if (!de) { - q_vdb_close_cat(cat_ctx); - return NULL; - } + if (cat_ctx->ctx->do_sort) { + if (cat_ctx->pkg_de == NULL) { + cat_ctx->pkg_cnt = scandirat(cat_ctx->fd, ".", &cat_ctx->pkg_de, + q_vdb_filter_pkg, cat_ctx->ctx->pkgsortfunc); + cat_ctx->pkg_cur = 0; + } + + while (cat_ctx->pkg_cur < cat_ctx->pkg_cnt) { + pkg_ctx = + q_vdb_open_pkg(cat_ctx, + cat_ctx->pkg_de[cat_ctx->pkg_cur++]->d_name); + if (!pkg_ctx) + continue; + break; + } + } else { + const struct dirent *de; + do { + de = readdir(cat_ctx->dir); + if (!de) + break; + + if (q_vdb_filter_pkg(de) == 0) + continue; - if (q_vdb_filter_pkg(de) == 0) - goto next_pkg; + pkg_ctx = q_vdb_open_pkg(cat_ctx, de->d_name); + if (!pkg_ctx) + continue; - pkg_ctx = q_vdb_open_pkg(cat_ctx, de->d_name); - if (!pkg_ctx) - goto next_pkg; + break; + } while (1); + } return pkg_ctx; } @@ -275,9 +322,10 @@ q_vdb_close_pkg(q_vdb_pkg_ctx *pkg_ctx) free(pkg_ctx); } -int -q_vdb_foreach_pkg(const char *sroot, const char *svdb, - q_vdb_pkg_cb callback, void *priv, q_vdb_cat_filter filter) +static int +q_vdb_foreach_pkg_int(const char *sroot, const char *svdb, + q_vdb_pkg_cb callback, void *priv, q_vdb_cat_filter filter, + bool sort, void *catsortfunc, void *pkgsortfunc) { q_vdb_ctx *ctx; q_vdb_cat_ctx *cat_ctx; @@ -288,6 +336,12 @@ q_vdb_foreach_pkg(const char *sroot, const char *svdb, if (!ctx) return EXIT_FAILURE; + ctx->do_sort = sort; + if (catsortfunc != NULL) + ctx->catsortfunc = catsortfunc; + if (pkgsortfunc != NULL) + ctx->pkgsortfunc = pkgsortfunc; + ret = 0; while ((cat_ctx = q_vdb_next_cat(ctx))) { if (filter && !filter(cat_ctx, priv)) @@ -296,53 +350,27 @@ q_vdb_foreach_pkg(const char *sroot, const char *svdb, ret |= callback(pkg_ctx, priv); q_vdb_close_pkg(pkg_ctx); } + q_vdb_close_cat(cat_ctx); } + q_vdb_close(ctx); return ret; } +int +q_vdb_foreach_pkg(const char *sroot, const char *svdb, + q_vdb_pkg_cb callback, void *priv, q_vdb_cat_filter filter) +{ + return q_vdb_foreach_pkg_int(sroot, svdb, callback, priv, + filter, false, NULL, NULL); +} + int q_vdb_foreach_pkg_sorted(const char *sroot, const char *svdb, q_vdb_pkg_cb callback, void *priv) { - q_vdb_ctx *ctx; - q_vdb_cat_ctx *cat_ctx; - q_vdb_pkg_ctx *pkg_ctx; - int ret = 0; - int c, p, cat_cnt, pkg_cnt; - struct dirent **cat_de, **pkg_de; - - ctx = q_vdb_open(sroot, svdb); - if (!ctx) - return EXIT_FAILURE; - - cat_cnt = scandirat(ctx->vdb_fd, ".", &cat_de, q_vdb_filter_cat, alphasort); - for (c = 0; c < cat_cnt; ++c) { - cat_ctx = q_vdb_open_cat(ctx, cat_de[c]->d_name); - if (!cat_ctx) - continue; - - pkg_cnt = scandirat(ctx->vdb_fd, cat_de[c]->d_name, &pkg_de, q_vdb_filter_pkg, alphasort); - for (p = 0; p < pkg_cnt; ++p) { - if (pkg_de[p]->d_name[0] == '-') - continue; - - pkg_ctx = q_vdb_open_pkg(cat_ctx, pkg_de[p]->d_name); - if (!pkg_ctx) - continue; - - ret |= callback(pkg_ctx, priv); - - q_vdb_close_pkg(pkg_ctx); - } - scandir_free(pkg_de, pkg_cnt); - - q_vdb_close_cat(cat_ctx); - } - scandir_free(cat_de, cat_cnt); - - q_vdb_close(ctx); - return ret; + return q_vdb_foreach_pkg_int(sroot, svdb, callback, priv, + NULL, true, NULL, NULL); } struct dirent * diff --git a/libq/vdb.h b/libq/vdb.h index 102f5a9f..ee2ee690 100644 --- a/libq/vdb.h +++ b/libq/vdb.h @@ -7,31 +7,59 @@ #define _VDB_H 1 #include +#include #include "set.h" +typedef struct q_vdb_ctx q_vdb_ctx; +typedef struct q_vdb_cat_ctx q_vdb_cat_ctx; +typedef struct q_vdb_pkg_ctx q_vdb_pkg_ctx; + /* VDB context */ -typedef struct { - int portroot_fd, vdb_fd; +struct q_vdb_ctx { + int portroot_fd; + int vdb_fd; DIR *dir; -} q_vdb_ctx; + struct dirent **cat_de; + size_t cat_cnt; + size_t cat_cur; + void *catsortfunc; + void *pkgsortfunc; + bool do_sort:1; + enum { + CACHE_UNSET = 0, + CACHE_METADATA_MD5, + CACHE_METADATA_PMS, + CACHE_EBUILD, + CACHE_VDB, + } cachetype:3; + q_vdb_pkg_ctx *ebuilddir_pkg_ctx; + q_vdb_cat_ctx *ebuilddir_cat_ctx; + q_vdb_ctx *ebuilddir_ctx; + char *repo; +}; /* Category context */ -typedef struct { +struct q_vdb_cat_ctx { const char *name; int fd; DIR *dir; const q_vdb_ctx *ctx; -} q_vdb_cat_ctx; + struct dirent **pkg_de; + size_t pkg_cnt; + size_t pkg_cur; +}; /* Package context */ -typedef struct { +struct q_vdb_pkg_ctx { const char *name; - char *slot, *repo; - size_t slot_len, repo_len; + char *slot; + char *repo; + size_t slot_len; + size_t repo_len; int fd; q_vdb_cat_ctx *cat_ctx; -} q_vdb_pkg_ctx; +}; /* Global helpers */ typedef int (q_vdb_pkg_cb)(q_vdb_pkg_ctx *, void *priv); diff --git a/qgrep.c b/qgrep.c index 9d78c18a..6cb56973 100644 --- a/qgrep.c +++ b/qgrep.c @@ -413,9 +413,9 @@ qgrep_cache_cb(cache_pkg_ctx *pkg_ctx, void *priv) /* need to construct path in portdir to ebuild, pass it to grep */ cctx = (cache_ctx *)(pkg_ctx->cat_ctx->ctx); if (cctx->cachetype == CACHE_EBUILD) { - pfd = cctx->dir_ctx->vdb_fd; + pfd = cctx->vdb_fd; } else { - pfd = openat(cctx->dir_ctx->vdb_fd, "../..", O_RDONLY|O_CLOEXEC); + pfd = openat(cctx->vdb_fd, "../..", O_RDONLY|O_CLOEXEC); } /* cat/pkg/pkg-ver.ebuild */ -- cgit v1.2.3-65-gdbad