aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMu Qiao <qiaomuf@gentoo.org>2011-07-01 17:51:38 +0800
committerMu Qiao <qiaomuf@gentoo.org>2011-07-05 09:49:27 +0800
commitd3296b24ca1fa7cafc5bf25e129a3c1ce0811ac7 (patch)
treeb1a08c3a06d0a05dee30a7cc0f4e173234e4e1c7
parentUtility: improve token printing (diff)
parentCore: fix memory leak (diff)
downloadlibbash-d3296b24ca1fa7cafc5bf25e129a3c1ce0811ac7.tar.gz
libbash-d3296b24ca1fa7cafc5bf25e129a3c1ce0811ac7.tar.bz2
libbash-d3296b24ca1fa7cafc5bf25e129a3c1ce0811ac7.zip
Merge remote branch 'betelgeuse/multithread' into multithreading
Conflicts: src/builtins/source_builtin.cpp src/core/bash_ast.cpp src/core/bash_ast.h utils/instruo.cpp
-rw-r--r--Makefile.am4
-rw-r--r--bashast/libbashWalker.g2
-rw-r--r--src/builtins/source_builtin.cpp51
-rw-r--r--src/core/bash_ast.cpp39
-rw-r--r--src/core/bash_ast.h8
-rw-r--r--utils/instruo.cpp112
6 files changed, 127 insertions, 89 deletions
diff --git a/Makefile.am b/Makefile.am
index fb36438..41b3e73 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -143,7 +143,7 @@ metadata_generator_CPPFLAGS = $(AM_CPPFLAGS) -Iutils
instruo_SOURCES = utils/instruo.cpp utils/command_line.cpp utils/command_line.h
instruo_LDADD = libcppbash.la @PALUDIS_LIBS@ libmetadata.a
instruo_CPPFLAGS = $(AM_CPPFLAGS) @PALUDIS_CFLAGS@ -Iutils
-instruo_CXXFLAGS = $(AM_CXXFLAGS) -Wno-extra
+instruo_CXXFLAGS = $(AM_CXXFLAGS) -Wno-extra -fopenmp
ast_printer_SOURCES = utils/ast_printer.cpp
ast_printer_LDADD = libcppbash.la $(BOOST_PROGRAM_OPTIONS_LIB)
@@ -367,7 +367,7 @@ massif: massif.out
callgrind.out: ast_printer
libtool --mode=execute valgrind \
--tool=callgrind --callgrind-out-file=callgrind.out \
- ./ast_printer -s -f /usr/portage/eclass/versionator.eclass
+ ./instruo -D /usr/portage/ --output-dir metadata_output
benchmark_parser: callgrind.out
callgrind_annotate callgrind.out
diff --git a/bashast/libbashWalker.g b/bashast/libbashWalker.g
index f379183..ef85ec6 100644
--- a/bashast/libbashWalker.g
+++ b/bashast/libbashWalker.g
@@ -56,7 +56,7 @@ options
@members{
- static interpreter* walker = 0;
+ static __thread interpreter* walker = 0;
void set_interpreter(interpreter* w)
{
diff --git a/src/builtins/source_builtin.cpp b/src/builtins/source_builtin.cpp
index bec83ad..d7924ab 100644
--- a/src/builtins/source_builtin.cpp
+++ b/src/builtins/source_builtin.cpp
@@ -27,6 +27,7 @@
#include <iostream>
#include <string>
#include <unordered_map>
+#include <thread>
#include "builtins/builtin_exceptions.h"
#include "cppbash_builtin.h"
@@ -34,35 +35,43 @@
#include "core/interpreter.h"
#include "core/bash_ast.h"
-int source_builtin::exec(const std::vector<std::string>& bash_args)
-{
- static std::unordered_map<std::string, std::shared_ptr<bash_ast>> ast_cache;
+namespace {
+ std::mutex parse_mutex;
- if(bash_args.size() == 0)
- throw libbash::illegal_argument_exception("should provide one argument for source builtin");
+ std::shared_ptr<bash_ast>& parse(const std::string& path)
+ {
+ static std::unordered_map<std::string, std::shared_ptr<bash_ast>> ast_cache;
- // we need fix this to pass extra arguments as positional parameters
- const std::string& path = bash_args[0];
+ std::lock_guard<std::mutex> parse_lock(parse_mutex);
- auto stored_ast = ast_cache.find(path);
- if(stored_ast == ast_cache.end())
- {
- // ensure the path is cached
- auto iter = ast_cache.insert(make_pair(path, std::shared_ptr<bash_ast>()));
- // this may throw exception
- iter.first->second.reset(new bash_ast(path));
- stored_ast = iter.first;
- }
- else if(!(stored_ast->second))
- {
- throw libbash::parse_exception(path + " cannot be fully parsed");
+ auto stored_ast = ast_cache.find(path);
+ if(stored_ast == ast_cache.end())
+ {
+ // ensure the path is cached
+ auto iter = ast_cache.insert(make_pair(path, std::shared_ptr<bash_ast>()));
+ // this may throw exception
+ iter.first->second.reset(new bash_ast(path));
+ stored_ast = iter.first;
+ }
+ else if(!(stored_ast->second))
+ {
+ throw libbash::parse_exception(path + " cannot be fully parsed");
+ }
+
+ return stored_ast->second;
}
+}
+
+int source_builtin::exec(const std::vector<std::string>& bash_args)
+{
+ if(bash_args.size() == 0)
+ throw libbash::illegal_argument_exception("should provide one argument for source builtin");
const std::string& original_path = _walker.resolve<std::string>("0");
+ _walker.define("0", bash_args.front(), true);
try
{
- _walker.define("0", path, true);
- stored_ast->second->interpret_with(_walker);
+ parse(bash_args.front())->interpret_with(_walker);
}
catch(return_exception& e) {}
diff --git a/src/core/bash_ast.cpp b/src/core/bash_ast.cpp
index c7b9180..3787f84 100644
--- a/src/core/bash_ast.cpp
+++ b/src/core/bash_ast.cpp
@@ -23,6 +23,8 @@
#include "core/bash_ast.h"
#include <fstream>
+#include <sstream>
+#include <thread>
#include <boost/numeric/conversion/cast.hpp>
@@ -54,6 +56,28 @@ bash_ast::bash_ast(const std::string& script_path,
init_parser(script, script_path);
}
+namespace
+{
+ std::mutex string_mutex;
+
+ pANTLR3_STRING locked_newRaw8(pANTLR3_STRING_FACTORY factory)
+ {
+ std::lock_guard<std::mutex> l(string_mutex);
+ pANTLR3_STRING_FACTORY pristine = antlr3StringFactoryNew();
+ pANTLR3_STRING result = pristine->newRaw(factory);
+ pristine->close(pristine);
+ return result;
+ }
+
+ void locked_destroy(pANTLR3_STRING_FACTORY factory, pANTLR3_STRING string)
+ {
+ std::lock_guard<std::mutex> l(string_mutex);
+ pANTLR3_STRING_FACTORY pristine = antlr3StringFactoryNew();
+ pristine->destroy(factory, string);
+ pristine->close(pristine);
+ }
+}
+
void bash_ast::init_parser(const std::string& script, const std::string& script_path)
{
input.reset(antlr3NewAsciiStringInPlaceStream(
@@ -83,13 +107,16 @@ void bash_ast::init_parser(const std::string& script, const std::string& script_
throw libbash::parse_exception("Out of memory trying to allocate parser");
ast = parse(parser.get());
+ ast->strFactory->newRaw = &locked_newRaw8;
+ ast->strFactory->destroy = &locked_destroy;
if(parser->pParser->rec->getNumberOfSyntaxErrors(parser->pParser->rec))
throw libbash::parse_exception("Something wrong happened while parsing");
- nodes.reset(antlr3CommonTreeNodeStreamNewTree(ast, ANTLR3_SIZE_HINT));
}
std::string bash_ast::get_dot_graph()
{
+ antlr_pointer<ANTLR3_COMMON_TREE_NODE_STREAM_struct> nodes(
+ antlr3CommonTreeNodeStreamNewTree(ast, ANTLR3_SIZE_HINT));
pANTLR3_STRING graph = nodes->adaptor->makeDot(nodes->adaptor, ast);
return std::string(reinterpret_cast<char*>(graph->chars));
}
@@ -154,6 +181,8 @@ std::string bash_ast::get_parser_tokens(antlr_pointer<ANTLR3_COMMON_TOKEN_STREAM
std::string bash_ast::get_walker_tokens(std::function<std::string(ANTLR3_UINT32)> token_map)
{
std::stringstream result;
+ antlr_pointer<ANTLR3_COMMON_TREE_NODE_STREAM_struct> nodes(
+ antlr3CommonTreeNodeStreamNewTree(ast, ANTLR3_SIZE_HINT));
pANTLR3_INT_STREAM istream = nodes->tnstream->istream;
auto istream_size = istream->size(istream);
@@ -196,6 +225,11 @@ void bash_ast::call_function(plibbashWalker ctx,
ANTLR3_MARKER index)
{
auto INPUT = ctx->pTreeParser->ctnstream;
+
+ // Initialize the input stream
+ auto ISTREAM = ctx->pTreeParser->ctnstream->tnstream->istream;
+ ISTREAM->size(ISTREAM);
+
// Push function index into INPUT
// The actual type of ANTLR3_MARKER is ANTLR3_INT32
INPUT->push(INPUT, boost::numeric_cast<ANTLR3_INT32>(index));
@@ -203,7 +237,8 @@ void bash_ast::call_function(plibbashWalker ctx,
ctx->compound_command(ctx);
}
-bash_ast::walker_pointer bash_ast::create_walker(interpreter& walker)
+bash_ast::walker_pointer bash_ast::create_walker(interpreter& walker,
+ antlr_pointer<ANTLR3_COMMON_TREE_NODE_STREAM_struct>& nodes)
{
set_interpreter(&walker);
walker.push_current_ast(this);
diff --git a/src/core/bash_ast.h b/src/core/bash_ast.h
index 22635a1..190b020 100644
--- a/src/core/bash_ast.h
+++ b/src/core/bash_ast.h
@@ -58,13 +58,13 @@ class bash_ast: public boost::noncopyable
antlr_pointer<ANTLR3_COMMON_TOKEN_STREAM_struct> token_stream;
antlr_pointer<libbashParser_Ctx_struct> parser;
pANTLR3_BASE_TREE ast;
- antlr_pointer<ANTLR3_COMMON_TREE_NODE_STREAM_struct> nodes;
std::function<pANTLR3_BASE_TREE(libbashParser_Ctx_struct*)> parse;
typedef std::unique_ptr<libbashWalker_Ctx_struct, std::function<void(plibbashWalker)>> walker_pointer;
void init_parser(const std::string& script, const std::string& script_path);
- walker_pointer create_walker(interpreter& walker);
+ walker_pointer create_walker(interpreter& walker,
+ antlr_pointer<ANTLR3_COMMON_TREE_NODE_STREAM_struct>& nodes);
public:
bash_ast(const std::istream& source,
@@ -92,7 +92,9 @@ public:
typename std::result_of<Functor(plibbashWalker)>::type
interpret_with(interpreter& walker, Functor walk)
{
- walker_pointer p_tree_parser = create_walker(walker);
+ antlr_pointer<ANTLR3_COMMON_TREE_NODE_STREAM_struct> nodes(
+ antlr3CommonTreeNodeStreamNewTree(ast, ANTLR3_SIZE_HINT));
+ walker_pointer p_tree_parser = create_walker(walker, nodes);
return walk(p_tree_parser.get());
}
diff --git a/utils/instruo.cpp b/utils/instruo.cpp
index dcd3924..704c98c 100644
--- a/utils/instruo.cpp
+++ b/utils/instruo.cpp
@@ -53,7 +53,7 @@
#include <paludis/package_database.hh>
#include <paludis/metadata_key.hh>
-#include "builtins/builtin_exceptions.h"
+#include "core/exceptions.h"
#include "command_line.h"
#include "libbash.h"
#include "utils/metadata.h"
@@ -65,71 +65,63 @@ using std::endl;
void worker(const std::shared_ptr<PackageIDSequence> &ids)
{
- std::unordered_map<std::string, std::vector<std::string>> variables;
-
- std::shared_ptr<const PackageID> id;
unsigned total(0);
CategoryNamePart old_cat("OLDCAT");
- while(!ids->empty())
+ #pragma omp parallel
{
- id = *ids->begin();
- ids->pop_front();
- if (id->name().category() != old_cat)
+ #pragma omp single nowait
+ while(!ids->empty())
{
- std::cout << "Processing " << stringify(id->name().category()) << "..." << std::endl;
- old_cat = id->name().category();
- FSPath(CommandLine::get_instance()->a_output_directory.argument() + "/" +
- stringify(id->name().category())).mkdir(0755, {fspmkdo_ok_if_exists});
- ++total;
- }
-
- Context i_context("When generating metadata for ID '" + stringify(*id) + "':");
-
- variables.clear();
- variables["PN"].push_back(stringify(id->name().package()));
- variables["PV"].push_back(stringify(id->version().remove_revision()));
- variables["P"].push_back(stringify(id->name().package()) + "-" +
- stringify(id->version().remove_revision()));
- variables["PR"].push_back(id->version().revision_only());
- variables["PVR"].push_back(stringify(id->version()));
- variables["PF"].push_back(stringify(id->name().package()) + "-" + stringify(id->version()));
- variables["CATEGORY"].push_back(stringify(id->name().category()));
- std::vector<std::string> functions;
+ std::shared_ptr<const PackageID> id = *ids->begin();
+ ids->pop_front();
+ if (id->name().category() != old_cat)
+ {
+ std::cout << "Processing " << stringify(id->name().category()) << "..." << std::endl;
+ old_cat = id->name().category();
+ FSPath(CommandLine::get_instance()->a_output_directory.argument() + "/" +
+ stringify(id->name().category())).mkdir(0755, {fspmkdo_ok_if_exists});
+ ++total;
+ }
- std::string ebuild_path(CommandLine::get_instance()->a_repository_directory.argument() +
- variables["CATEGORY"][0] + "/" +
- variables["PN"][0] + "/" +
- variables["PN"][0] + "-" +
- variables["PVR"][0] + ".ebuild");
- try
- {
- libbash::interpret(ebuild_path, "utils/isolated-functions.sh", variables, functions);
- }
- catch(const libbash::interpreter_exception& e)
- {
- cerr << "Exception occurred while interpreting " << ebuild_path << ". The error message is:\n"
- << e.what() << endl;
- continue;
- }
- catch(const return_exception& e)
- {
- cerr << "Unhandled return exception in " << ebuild_path << ". The error message is:\n"
- << e.what() << endl;
- continue;
- }
- catch (...)
- {
- cerr << "Unhandled exception in " << ebuild_path << endl;
- continue;
+ #pragma omp task
+ {
+ Context i_context("When generating metadata for ID '" + stringify(*id) + "':");
+
+ std::unordered_map<std::string, std::vector<std::string>> variables;
+ variables["PN"].push_back(stringify(id->name().package()));
+ variables["PV"].push_back(stringify(id->version().remove_revision()));
+ variables["P"].push_back(stringify(id->name().package()) + "-" +
+ stringify(id->version().remove_revision()));
+ variables["PR"].push_back(id->version().revision_only());
+ variables["PVR"].push_back(stringify(id->version()));
+ variables["PF"].push_back(stringify(id->name().package()) + "-" + stringify(id->version()));
+ variables["CATEGORY"].push_back(stringify(id->name().category()));
+ std::vector<std::string> functions;
+
+ std::string ebuild_path(CommandLine::get_instance()->a_repository_directory.argument() +
+ variables["CATEGORY"][0] + "/" +
+ variables["PN"][0] + "/" +
+ variables["PN"][0] + "-" +
+ variables["PVR"][0] + ".ebuild");
+ try
+ {
+ libbash::interpret(ebuild_path, variables, functions);
+
+ std::string output_path(CommandLine::get_instance()->a_output_directory.argument() + "/" +
+ variables["CATEGORY"][0] + "/" +
+ variables["PN"][0] + "-" +
+ variables["PVR"][0]);
+ FSPath(output_path).dirname().mkdir(0755, {fspmkdo_ok_if_exists});
+ std::ofstream output(output_path, std::ofstream::out | std::ofstream::trunc);
+ write_metadata(output, variables, functions);
+ }
+ catch(const libbash::interpreter_exception& e)
+ {
+ cerr << "Exception occurred while interpreting " << ebuild_path << ". The error message is:\n"
+ << e.what() << endl;
+ }
+ }
}
-
- std::string output_path(CommandLine::get_instance()->a_output_directory.argument() + "/" +
- variables["CATEGORY"][0] + "/" +
- variables["PN"][0] + "-" +
- variables["PVR"][0]);
- FSPath(output_path).dirname().mkdir(0755, {fspmkdo_ok_if_exists});
- std::ofstream output(output_path, std::ofstream::out | std::ofstream::trunc);
- write_metadata(output, variables, functions);
}
}