libq/tree: have tree_foreach_pkg take a query atom

Allow to reduce the search by having a query atom.  This will skip
categories and packages not matching the atom, possibly avoiding lots of
work.

This needs to be exploited from applets where necessary.

Signed-off-by: Fabian Groffen <grobian@gentoo.org>

11 files changed, 103 insertions, 92 deletions

diff --git a/TODO.md b/TODO.md
index d89dda1c..71a0ce15 100644
--- a/TODO.md
+++ b/TODO.md
@@ -9,8 +9,6 @@
   `ACCEPT_LICENSE="${ACCEPT_LICENSE} bar"`<br>
   we end up getting just:<br>
   `ACCEPT_LICENSE=" bar"`
-- tree\_foreach\_pkg should have variant that takes an atom (or just
-  cat?) to reduce search space
 - tree\_get\_atoms should return atoms iso string set, needs a rewrite
   to use foreach\_pkg and get\_atom -- set is ready for storing objects
   now
diff --git a/libq/tree.c b/libq/tree.c
index 377af83d..8996e550 100644
--- a/libq/tree.c
+++ b/libq/tree.c
@@ -57,8 +57,6 @@ tree_open_int(const char *sroot, const char *tdir, bool quiet)
 		goto cv_error;
 
 	ctx->do_sort = false;
-	ctx->catsortfunc = alphasort;
-	ctx->pkgsortfunc = tree_pkg_compar;
 	return ctx;
 
  cv_error:
@@ -237,23 +235,31 @@ tree_next_cat(tree_ctx *ctx)
 {
 	/* search for a category directory */
 	tree_cat_ctx *cat_ctx = NULL;
+	const struct dirent *de;
 
 	if (ctx->do_sort) {
 		if (ctx->cat_de == NULL) {
 			ctx->cat_cnt = scandirat(ctx->tree_fd,
-					".", &ctx->cat_de, tree_filter_cat, ctx->catsortfunc);
+					".", &ctx->cat_de, tree_filter_cat, alphasort);
 			ctx->cat_cur = 0;
 		}
 
 		while (ctx->cat_cur < ctx->cat_cnt) {
-			cat_ctx = tree_open_cat(ctx, ctx->cat_de[ctx->cat_cur++]->d_name);
+			de = ctx->cat_de[ctx->cat_cur++];
+
+			/* match if cat is requested */
+			if (ctx->query_atom != NULL && ctx->query_atom->CATEGORY != NULL &&
+					strcmp(ctx->query_atom->CATEGORY, de->d_name) != 0)
+				continue;
+
+			cat_ctx = tree_open_cat(ctx, de->d_name);
 			if (!cat_ctx)
 				continue;
+
 			break;
 		}
 	} else {
 		/* cheaper "streaming" variant */
-		const struct dirent *de;
 		do {
 			de = readdir(ctx->dir);
 			if (!de)
@@ -262,6 +268,11 @@ tree_next_cat(tree_ctx *ctx)
 			if (tree_filter_cat(de) == 0)
 				continue;
 
+			/* match if cat is requested */
+			if (ctx->query_atom != NULL && ctx->query_atom->CATEGORY != NULL &&
+					strcmp(ctx->query_atom->CATEGORY, de->d_name) != 0)
+				continue;
+
 			cat_ctx = tree_open_cat(ctx, de->d_name);
 			if (!cat_ctx)
 				continue;
@@ -327,6 +338,17 @@ tree_open_pkg(tree_cat_ctx *cat_ctx, const char *name)
 	pkg_ctx->cat_ctx = cat_ctx;
 	pkg_ctx->atom = NULL;
 	pkg_ctx->meta = NULL;
+
+	/* see if this pkg matches the query, here we can finally check
+	 * version conditions like >=, etc. */
+	if (cat_ctx->ctx->query_atom != NULL) {
+		(void)tree_get_atom(pkg_ctx, false);
+		if (atom_compare(pkg_ctx->atom, cat_ctx->ctx->query_atom) != EQUAL) {
+			tree_close_pkg(pkg_ctx);
+			return NULL;
+		}
+	}
+
 	return pkg_ctx;
 }
 
@@ -348,6 +370,7 @@ tree_next_pkg_int(tree_cat_ctx *cat_ctx)
 {
 	tree_pkg_ctx *pkg_ctx = NULL;
 	const struct dirent *de;
+	depend_atom *qa = cat_ctx->ctx->query_atom;
 
 	if (cat_ctx->ctx->do_sort) {
 		if (cat_ctx->pkg_ctxs == NULL) {
@@ -360,11 +383,22 @@ tree_next_pkg_int(tree_cat_ctx *cat_ctx)
 				if (tree_filter_pkg(de) == 0)
 					continue;
 
+				/* perform package name check, for we don't have an atom
+				 * yet, and creating it is expensive, which we better
+				 * defer to pkg time, and filter most stuff out here
+				 * note that we might over-match, but that's easier than
+				 * trying to deal with end of string or '-' here (which
+				 * still wouldn't be 100% because name rules are complex) */
+				if (qa != NULL && qa->PN != NULL &&
+						strncmp(qa->PN, de->d_name, strlen(qa->PN)) != 0)
+					continue;
+
 				if (cat_ctx->pkg_cnt == pkg_size) {
 					pkg_size += 256;
 					cat_ctx->pkg_ctxs = xrealloc(cat_ctx->pkg_ctxs,
 								sizeof(*cat_ctx->pkg_ctxs) * pkg_size);
 				}
+
 				name = xstrdup(de->d_name);
 				pkg_ctx = cat_ctx->pkg_ctxs[cat_ctx->pkg_cnt++] =
 					tree_open_pkg(cat_ctx, name);
@@ -374,9 +408,9 @@ tree_next_pkg_int(tree_cat_ctx *cat_ctx)
 				}
 			}
 
-			if (cat_ctx->ctx->pkgsortfunc != NULL && cat_ctx->pkg_cnt > 1) {
+			if (cat_ctx->pkg_cnt > 1) {
 				qsort(cat_ctx->pkg_ctxs, cat_ctx->pkg_cnt,
-						sizeof(*cat_ctx->pkg_ctxs), cat_ctx->ctx->pkgsortfunc);
+						sizeof(*cat_ctx->pkg_ctxs), tree_pkg_compar);
 			}
 		}
 
@@ -392,6 +426,11 @@ tree_next_pkg_int(tree_cat_ctx *cat_ctx)
 			if (tree_filter_pkg(de) == 0)
 				continue;
 
+			/* perform package name check as for the sorted variant */
+			if (qa != NULL && qa->PN != NULL &&
+					strncmp(qa->PN, de->d_name, strlen(qa->PN)) != 0)
+				continue;
+
 			pkg_ctx = tree_open_pkg(cat_ctx, de->d_name);
 			if (!pkg_ctx)
 				continue;
@@ -428,8 +467,6 @@ tree_next_pkg(tree_cat_ctx *cat_ctx)
 				pkgdir->portroot_fd = -1;
 				pkgdir->tree_fd = cat_ctx->fd;
 				pkgdir->do_sort = ctx->do_sort;
-				pkgdir->catsortfunc = ctx->catsortfunc;
-				pkgdir->pkgsortfunc = ctx->pkgsortfunc;
 				pkgdir->repo = ctx->repo;
 				pkgdir->cachetype = ctx->cachetype;
 
@@ -440,7 +477,7 @@ tree_next_pkg(tree_cat_ctx *cat_ctx)
 				 * directory or something */
 				if (ctx->ebuilddir_cat_ctx == NULL) {
 					ctx->ebuilddir_pkg_ctx = NULL;
-					return NULL;
+					continue;
 				}
 
 				/* "zap" the pkg such that it looks like CAT/P */
@@ -1155,7 +1192,8 @@ tree_close_pkg(tree_pkg_ctx *pkg_ctx)
 }
 
 static int
-tree_foreach_packages(tree_ctx *ctx, tree_pkg_cb callback, void *priv)
+tree_foreach_packages(tree_ctx *ctx, tree_pkg_cb callback,
+		void *priv, depend_atom *query)
 {
 	char *p = ctx->pkgs;
 	char *q;
@@ -1239,6 +1277,11 @@ tree_foreach_packages(tree_ctx *ctx, tree_pkg_cb callback, void *priv)
 			if (atom != NULL)
 				atom_implode(atom);
 			atom = atom_explode(c);
+			/* pretend this entry is bogus if it doesn't match query */
+			if (query != NULL && atom_compare(atom, query) != EQUAL) {
+				atom_implode(atom);
+				atom = NULL;
+			}
 #define match_key(X) match_key2(X,X)
 #define match_key2(X,Y) \
 		} else if (strcmp(p, #X) == 0) { \
@@ -1270,9 +1313,8 @@ tree_foreach_packages(tree_ctx *ctx, tree_pkg_cb callback, void *priv)
 }
 
 int
-tree_foreach_pkg(tree_ctx *ctx,
-		tree_pkg_cb callback, void *priv, tree_cat_filter filter,
-		bool sort, void *catsortfunc, void *pkgsortfunc)
+tree_foreach_pkg(tree_ctx *ctx, tree_pkg_cb callback, void *priv,
+		bool sort, depend_atom *query)
 {
 	tree_cat_ctx *cat_ctx;
 	tree_pkg_ctx *pkg_ctx;
@@ -1283,18 +1325,12 @@ tree_foreach_pkg(tree_ctx *ctx,
 
 	/* handle Packages (binpkgs index) file separately */
 	if (ctx->cachetype == CACHE_PACKAGES)
-		return tree_foreach_packages(ctx, callback, priv);
+		return tree_foreach_packages(ctx, callback, priv, query);
 
 	ctx->do_sort = sort;
-	if (catsortfunc != NULL)
-		ctx->catsortfunc = catsortfunc;
-	if (pkgsortfunc != NULL)
-		ctx->pkgsortfunc = pkgsortfunc;
 
 	ret = 0;
 	while ((cat_ctx = tree_next_cat(ctx))) {
-		if (filter && !filter(cat_ctx, priv))
-			continue;
 		while ((pkg_ctx = tree_next_pkg(cat_ctx))) {
 			ret |= callback(pkg_ctx, priv);
 			tree_close_pkg(pkg_ctx);
@@ -1319,6 +1355,7 @@ tree_get_atom(tree_pkg_ctx *pkg_ctx, bool complete)
 		tree_ctx *ctx = pkg_ctx->cat_ctx->ctx;
 		if (ctx->cachetype == CACHE_VDB) {
 			if (pkg_ctx->atom->SLOT == NULL) {
+				/* FIXME: use tree_meta_get !!! */
 				if (pkg_ctx->slot == NULL)
 					tree_pkg_vdb_eat(pkg_ctx, "SLOT",
 							&pkg_ctx->slot, &pkg_ctx->slot_len);
diff --git a/libq/tree.h b/libq/tree.h
index 8a37cbbc..eb60296e 100644
--- a/libq/tree.h
+++ b/libq/tree.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2019 Gentoo Foundation
+ * Copyright 2005-2020 Gentoo Foundation
  * Distributed under the terms of the GNU General Public License v2
  */
 
@@ -27,8 +27,6 @@ struct tree_ctx {
 	struct dirent **cat_de;
 	size_t cat_cnt;
 	size_t cat_cur;
-	void *catsortfunc;
-	void *pkgsortfunc;
 	bool do_sort:1;
 	enum {
 		CACHE_UNSET = 0,
@@ -45,6 +43,7 @@ struct tree_ctx {
 	char *repo;
 	char *pkgs;
 	size_t pkgslen;
+	depend_atom *query_atom;
 };
 
 /* Category context */
@@ -110,9 +109,8 @@ struct tree_metadata_xml {
 	} *email;
 };
 
-/* Global helpers */
+/* foreach pkg callback function signature */
 typedef int (tree_pkg_cb)(tree_pkg_ctx *, void *priv);
-typedef int (tree_cat_filter)(tree_cat_ctx *, void *priv);
 
 tree_ctx *tree_open(const char *sroot, const char *portdir);
 tree_ctx *tree_open_vdb(const char *sroot, const char *svdb);
@@ -140,13 +138,12 @@ char *tree_pkg_meta_get_int(tree_pkg_ctx *pkg_ctx, size_t offset, const char *ke
 tree_metadata_xml *tree_pkg_metadata(tree_pkg_ctx *pkg_ctx);
 void tree_close_metadata(tree_metadata_xml *meta_ctx);
 void tree_close_pkg(tree_pkg_ctx *pkg_ctx);
-int tree_foreach_pkg(tree_ctx *ctx,
-		tree_pkg_cb callback, void *priv, tree_cat_filter filter,
-		bool sort, void *catsortfunc, void *pkgsortfunc);
-#define tree_foreach_pkg_fast(ctx, cb, priv, filter) \
-	tree_foreach_pkg(ctx, cb, priv, filter, false, NULL, NULL);
-#define tree_foreach_pkg_sorted(ctx, cb, priv) \
-	tree_foreach_pkg(ctx, cb, priv, NULL, true, NULL, NULL);
+int tree_foreach_pkg(tree_ctx *ctx, tree_pkg_cb callback, void *priv,
+		bool sort, depend_atom *query);
+#define tree_foreach_pkg_fast(ctx, cb, priv, query) \
+	tree_foreach_pkg(ctx, cb, priv, false, query);
+#define tree_foreach_pkg_sorted(ctx, cb, priv, query) \
+	tree_foreach_pkg(ctx, cb, priv, true, query);
 struct dirent *tree_get_next_dir(DIR *dir);
 set *tree_get_atoms(tree_ctx *ctx, bool fullcpv, set *satoms);
 depend_atom *tree_get_atom(tree_pkg_ctx *pkg_ctx, bool complete);
diff --git a/qcheck.c b/qcheck.c
index ab4356bb..66a4ee78 100644
--- a/qcheck.c
+++ b/qcheck.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2019 Gentoo Foundation
+ * Copyright 2005-2020 Gentoo Foundation
  * Distributed under the terms of the GNU General Public License v2
  *
  * Copyright 2005-2010 Ned Ludd        - <solar@gentoo.org>
@@ -434,7 +434,7 @@ int qcheck_main(int argc, char **argv)
 	vdb = tree_open_vdb(portroot, portvdb);
 	ret = -1;
 	if (vdb != NULL) {
-		ret = tree_foreach_pkg_sorted(vdb, qcheck_cb, &state);
+		ret = tree_foreach_pkg_sorted(vdb, qcheck_cb, &state, NULL);
 		tree_close(vdb);
 	}
 	if (array_cnt(regex_arr) > 0) {
diff --git a/qdepends.c b/qdepends.c
index 9969d905..44821b6f 100644
--- a/qdepends.c
+++ b/qdepends.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2019 Gentoo Authors
+ * Copyright 2005-2020 Gentoo Authors
  * Distributed under the terms of the GNU General Public License v2
  *
  * Copyright 2005-2010 Ned Ludd        - <solar@gentoo.org>
@@ -376,7 +376,7 @@ int qdepends_main(int argc, char **argv)
 			t = tree_open(portroot, overlay);
 			if (t != NULL) {
 				ret = tree_foreach_pkg_sorted(t,
-						qdepends_results_cb, &state);
+						qdepends_results_cb, &state, NULL);
 				tree_close(t);
 			}
 		}
diff --git a/qfile.c b/qfile.c
index b1aab8d1..0b01061f 100644
--- a/qfile.c
+++ b/qfile.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2019 Gentoo Foundation
+ * Copyright 2005-2020 Gentoo Foundation
  * Distributed under the terms of the GNU General Public License v2
  *
  * Copyright 2005-2010 Ned Ludd        - <solar@gentoo.org>
@@ -569,7 +569,7 @@ int qfile_main(int argc, char **argv)
 	if (nb_of_queries > 0) {
 		tree_ctx *vdb = tree_open_vdb(portroot, portvdb);
 		if (vdb != NULL) {
-			found += tree_foreach_pkg_sorted(vdb, qfile_cb, &state);
+			found += tree_foreach_pkg_sorted(vdb, qfile_cb, &state, NULL);
 			tree_close(vdb);
 		}
 	}
diff --git a/qkeyword.c b/qkeyword.c
index 70f75ded..f7779602 100644
--- a/qkeyword.c
+++ b/qkeyword.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2019 Gentoo Foundation
+ * Copyright 2005-2020 Gentoo Foundation
  * Distributed under the terms of the GNU General Public License v2
  *
  * Copyright 2006      Thomas A. Cort - <tcort@gentoo.org>
@@ -799,7 +799,7 @@ qkeyword_traverse(tree_pkg_cb func, void *priv)
 	array_for_each(overlays, n, overlay) {
 		tree_ctx *t = tree_open(portroot, overlay);
 		if (t != NULL) {
-			ret |= tree_foreach_pkg_sorted(t, qkeyword_results_cb, priv);
+			ret |= tree_foreach_pkg_sorted(t, qkeyword_results_cb, priv, NULL);
 			tree_close(t);
 		}
 	}
diff --git a/qlist.c b/qlist.c
index f77eaea8..12d63f83 100644
--- a/qlist.c
+++ b/qlist.c
@@ -494,7 +494,7 @@ int qlist_main(int argc, char **argv)
 	else
 		vdb = tree_open_vdb(portroot, portvdb);
 	if (vdb != NULL) {
-		ret = tree_foreach_pkg_sorted(vdb, qlist_cb, &state);
+		ret = tree_foreach_pkg_sorted(vdb, qlist_cb, &state, NULL);
 		tree_close(vdb);
 	}
 	free(state.buf);
diff --git a/qmerge.c b/qmerge.c
index a2d06ab3..6d9439f1 100644
--- a/qmerge.c
+++ b/qmerge.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2019 Gentoo Authors
+ * Copyright 2005-2020 Gentoo Authors
  * Distributed under the terms of the GNU General Public License v2
  *
  * Copyright 2005-2010 Ned Ludd        - <solar@gentoo.org>
@@ -272,37 +272,15 @@ qmerge_initialize(void)
 	free(buf);
 }
 
-struct qmerge_bv_state {
-	const char *catname;
-	const char *pkgname;
-	const char *slot;
-	char buf[4096];
-	char *retbuf;
-};
-
-static int
-qmerge_filter_cat(tree_cat_ctx *cat_ctx, void *priv)
-{
-	struct qmerge_bv_state *state = priv;
-	return !state->catname || strcmp(cat_ctx->name, state->catname) == 0;
-}
-
-/* HACK: pull this in, knowing that qlist will be in the final link, we
- * should however figure out how to do what match does here from e.g.
- * atom */
-extern bool qlist_match(
-		tree_pkg_ctx *pkg_ctx,
-		const char *name,
-		depend_atom **name_atom,
-		bool exact);
-
+static char _best_version_retbuf[4096];
 static int
 qmerge_best_version_cb(tree_pkg_ctx *pkg_ctx, void *priv)
 {
-	struct qmerge_bv_state *state = priv;
-	if (qlist_match(pkg_ctx, state->buf, NULL, true))
-		snprintf(state->retbuf, sizeof(state->buf), "%s/%s:%s",
-			 pkg_ctx->cat_ctx->name, pkg_ctx->name, state->slot);
+	depend_atom *sa = priv;
+	depend_atom *a = tree_get_atom(pkg_ctx, true);  /* need SLOT */
+	if (atom_compare(a, sa) == EQUAL)
+		snprintf(_best_version_retbuf, sizeof(_best_version_retbuf),
+				"%s/%s:%s", a->CATEGORY, a->PF, a->SLOT);
 	return 0;
 }
 
@@ -310,15 +288,7 @@ static char *
 best_version(const char *catname, const char *pkgname, const char *slot)
 {
 	static int vdb_check = 1;
-	static char retbuf[4096];
-
 	tree_ctx *vdb;
-	struct qmerge_bv_state state = {
-		.catname = catname,
-		.pkgname = pkgname,
-		.slot = slot,
-		.retbuf = retbuf,
-	};
 
 	/* Make sure these dirs exist before we try walking them */
 	switch (vdb_check) {
@@ -336,18 +306,18 @@ best_version(const char *catname, const char *pkgname, const char *slot)
 			goto done;
 	}
 
-	retbuf[0] = '\0';
-	snprintf(state.buf, sizeof(state.buf), "%s%s%s:%s",
-		 catname ? : "", catname ? "/" : "", pkgname, slot);
+	snprintf(_best_version_retbuf, sizeof(_best_version_retbuf),
+			"%s%s%s:%s", catname ? : "", catname ? "/" : "", pkgname, slot);
 	vdb = tree_open_vdb(portroot, portvdb);
 	if (vdb != NULL) {
-		tree_foreach_pkg_fast(vdb,
-				qmerge_best_version_cb, &state, qmerge_filter_cat);
+		depend_atom *sa = atom_explode(_best_version_retbuf);
+		tree_foreach_pkg_fast(vdb, qmerge_best_version_cb, sa, sa);
 		tree_close(vdb);
+		atom_implode(sa);
 	}
 
  done:
-	return retbuf;
+	return _best_version_retbuf;
 }
 
 static int
@@ -1810,6 +1780,15 @@ print_Pkg(int full, const depend_atom *atom, const struct pkg_t *pkg)
 	}
 }
 
+/* HACK: pull this in, knowing that qlist will be in the final link, we
+ * should however figure out how to do what match does here from e.g.
+ * atom */
+extern bool qlist_match(
+		tree_pkg_ctx *pkg_ctx,
+		const char *name,
+		depend_atom **name_atom,
+		bool exact);
+
 static int
 qmerge_unmerge_cb(tree_pkg_ctx *pkg_ctx, void *priv)
 {
diff --git a/qsearch.c b/qsearch.c
index a26e2d6b..31b183c3 100644
--- a/qsearch.c
+++ b/qsearch.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2019 Gentoo Authors
+ * Copyright 2005-2020 Gentoo Authors
  * Distributed under the terms of the GNU General Public License v2
  *
  * Copyright 2005-2010 Ned Ludd        - <solar@gentoo.org>
@@ -174,7 +174,7 @@ int qsearch_main(int argc, char **argv)
 	array_for_each(overlays, n, overlay) {
 		tree_ctx *t = tree_open(portroot, overlay);
 		if (t != NULL) {
-			ret |= tree_foreach_pkg_sorted(t, qsearch_cb, &state);
+			ret |= tree_foreach_pkg_sorted(t, qsearch_cb, &state, NULL);
 			tree_close(t);
 		}
 	}
diff --git a/quse.c b/quse.c
index be34e48b..a8585bef 100644
--- a/quse.c
+++ b/quse.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2019 Gentoo Foundation
+ * Copyright 2005-2020 Gentoo Foundation
  * Distributed under the terms of the GNU General Public License v2
  *
  * Copyright 2005-2010 Ned Ludd        - <solar@gentoo.org>
@@ -719,14 +719,14 @@ int quse_main(int argc, char **argv)
 	} else if (state.do_installed) {
 		tree_ctx *t = tree_open_vdb(portroot, portvdb);
 		state.overlay = NULL;
-		tree_foreach_pkg_sorted(t, quse_results_cb, &state);
+		tree_foreach_pkg_sorted(t, quse_results_cb, &state, NULL);
 		tree_close(t);
 	} else {
 		array_for_each(overlays, n, overlay) {
 			tree_ctx *t = tree_open(portroot, overlay);
 			state.overlay = overlay;
 			if (t != NULL) {
-				tree_foreach_pkg_sorted(t, quse_results_cb, &state);
+				tree_foreach_pkg_sorted(t, quse_results_cb, &state, NULL);
 				tree_close(t);
 			}
 		}