diff options
| -rw-r--r-- | Makefile.am | 4 | ||||
| -rw-r--r-- | bashast/libbashWalker.g | 2 | ||||
| -rw-r--r-- | src/builtins/source_builtin.cpp | 51 | ||||
| -rw-r--r-- | src/core/bash_ast.cpp | 39 | ||||
| -rw-r--r-- | src/core/bash_ast.h | 8 | ||||
| -rw-r--r-- | utils/instruo.cpp | 112 |
6 files changed, 127 insertions, 89 deletions
diff --git a/Makefile.am b/Makefile.am index fb36438..41b3e73 100644 --- a/Makefile.am +++ b/Makefile.am @@ -143,7 +143,7 @@ metadata_generator_CPPFLAGS = $(AM_CPPFLAGS) -Iutils instruo_SOURCES = utils/instruo.cpp utils/command_line.cpp utils/command_line.h instruo_LDADD = libcppbash.la @PALUDIS_LIBS@ libmetadata.a instruo_CPPFLAGS = $(AM_CPPFLAGS) @PALUDIS_CFLAGS@ -Iutils -instruo_CXXFLAGS = $(AM_CXXFLAGS) -Wno-extra +instruo_CXXFLAGS = $(AM_CXXFLAGS) -Wno-extra -fopenmp ast_printer_SOURCES = utils/ast_printer.cpp ast_printer_LDADD = libcppbash.la $(BOOST_PROGRAM_OPTIONS_LIB) @@ -367,7 +367,7 @@ massif: massif.out callgrind.out: ast_printer libtool --mode=execute valgrind \ --tool=callgrind --callgrind-out-file=callgrind.out \ - ./ast_printer -s -f /usr/portage/eclass/versionator.eclass + ./instruo -D /usr/portage/ --output-dir metadata_output benchmark_parser: callgrind.out callgrind_annotate callgrind.out diff --git a/bashast/libbashWalker.g b/bashast/libbashWalker.g index f379183..ef85ec6 100644 --- a/bashast/libbashWalker.g +++ b/bashast/libbashWalker.g @@ -56,7 +56,7 @@ options @members{ - static interpreter* walker = 0; + static __thread interpreter* walker = 0; void set_interpreter(interpreter* w) { diff --git a/src/builtins/source_builtin.cpp b/src/builtins/source_builtin.cpp index bec83ad..d7924ab 100644 --- a/src/builtins/source_builtin.cpp +++ b/src/builtins/source_builtin.cpp @@ -27,6 +27,7 @@ #include <iostream> #include <string> #include <unordered_map> +#include <thread> #include "builtins/builtin_exceptions.h" #include "cppbash_builtin.h" @@ -34,35 +35,43 @@ #include "core/interpreter.h" #include "core/bash_ast.h" -int source_builtin::exec(const std::vector<std::string>& bash_args) -{ - static std::unordered_map<std::string, std::shared_ptr<bash_ast>> ast_cache; +namespace { + std::mutex parse_mutex; - if(bash_args.size() == 0) - throw libbash::illegal_argument_exception("should provide one argument for source builtin"); + std::shared_ptr<bash_ast>& parse(const std::string& path) + { + static std::unordered_map<std::string, std::shared_ptr<bash_ast>> ast_cache; - // we need fix this to pass extra arguments as positional parameters - const std::string& path = bash_args[0]; + std::lock_guard<std::mutex> parse_lock(parse_mutex); - auto stored_ast = ast_cache.find(path); - if(stored_ast == ast_cache.end()) - { - // ensure the path is cached - auto iter = ast_cache.insert(make_pair(path, std::shared_ptr<bash_ast>())); - // this may throw exception - iter.first->second.reset(new bash_ast(path)); - stored_ast = iter.first; - } - else if(!(stored_ast->second)) - { - throw libbash::parse_exception(path + " cannot be fully parsed"); + auto stored_ast = ast_cache.find(path); + if(stored_ast == ast_cache.end()) + { + // ensure the path is cached + auto iter = ast_cache.insert(make_pair(path, std::shared_ptr<bash_ast>())); + // this may throw exception + iter.first->second.reset(new bash_ast(path)); + stored_ast = iter.first; + } + else if(!(stored_ast->second)) + { + throw libbash::parse_exception(path + " cannot be fully parsed"); + } + + return stored_ast->second; } +} + +int source_builtin::exec(const std::vector<std::string>& bash_args) +{ + if(bash_args.size() == 0) + throw libbash::illegal_argument_exception("should provide one argument for source builtin"); const std::string& original_path = _walker.resolve<std::string>("0"); + _walker.define("0", bash_args.front(), true); try { - _walker.define("0", path, true); - stored_ast->second->interpret_with(_walker); + parse(bash_args.front())->interpret_with(_walker); } catch(return_exception& e) {} diff --git a/src/core/bash_ast.cpp b/src/core/bash_ast.cpp index c7b9180..3787f84 100644 --- a/src/core/bash_ast.cpp +++ b/src/core/bash_ast.cpp @@ -23,6 +23,8 @@ #include "core/bash_ast.h" #include <fstream> +#include <sstream> +#include <thread> #include <boost/numeric/conversion/cast.hpp> @@ -54,6 +56,28 @@ bash_ast::bash_ast(const std::string& script_path, init_parser(script, script_path); } +namespace +{ + std::mutex string_mutex; + + pANTLR3_STRING locked_newRaw8(pANTLR3_STRING_FACTORY factory) + { + std::lock_guard<std::mutex> l(string_mutex); + pANTLR3_STRING_FACTORY pristine = antlr3StringFactoryNew(); + pANTLR3_STRING result = pristine->newRaw(factory); + pristine->close(pristine); + return result; + } + + void locked_destroy(pANTLR3_STRING_FACTORY factory, pANTLR3_STRING string) + { + std::lock_guard<std::mutex> l(string_mutex); + pANTLR3_STRING_FACTORY pristine = antlr3StringFactoryNew(); + pristine->destroy(factory, string); + pristine->close(pristine); + } +} + void bash_ast::init_parser(const std::string& script, const std::string& script_path) { input.reset(antlr3NewAsciiStringInPlaceStream( @@ -83,13 +107,16 @@ void bash_ast::init_parser(const std::string& script, const std::string& script_ throw libbash::parse_exception("Out of memory trying to allocate parser"); ast = parse(parser.get()); + ast->strFactory->newRaw = &locked_newRaw8; + ast->strFactory->destroy = &locked_destroy; if(parser->pParser->rec->getNumberOfSyntaxErrors(parser->pParser->rec)) throw libbash::parse_exception("Something wrong happened while parsing"); - nodes.reset(antlr3CommonTreeNodeStreamNewTree(ast, ANTLR3_SIZE_HINT)); } std::string bash_ast::get_dot_graph() { + antlr_pointer<ANTLR3_COMMON_TREE_NODE_STREAM_struct> nodes( + antlr3CommonTreeNodeStreamNewTree(ast, ANTLR3_SIZE_HINT)); pANTLR3_STRING graph = nodes->adaptor->makeDot(nodes->adaptor, ast); return std::string(reinterpret_cast<char*>(graph->chars)); } @@ -154,6 +181,8 @@ std::string bash_ast::get_parser_tokens(antlr_pointer<ANTLR3_COMMON_TOKEN_STREAM std::string bash_ast::get_walker_tokens(std::function<std::string(ANTLR3_UINT32)> token_map) { std::stringstream result; + antlr_pointer<ANTLR3_COMMON_TREE_NODE_STREAM_struct> nodes( + antlr3CommonTreeNodeStreamNewTree(ast, ANTLR3_SIZE_HINT)); pANTLR3_INT_STREAM istream = nodes->tnstream->istream; auto istream_size = istream->size(istream); @@ -196,6 +225,11 @@ void bash_ast::call_function(plibbashWalker ctx, ANTLR3_MARKER index) { auto INPUT = ctx->pTreeParser->ctnstream; + + // Initialize the input stream + auto ISTREAM = ctx->pTreeParser->ctnstream->tnstream->istream; + ISTREAM->size(ISTREAM); + // Push function index into INPUT // The actual type of ANTLR3_MARKER is ANTLR3_INT32 INPUT->push(INPUT, boost::numeric_cast<ANTLR3_INT32>(index)); @@ -203,7 +237,8 @@ void bash_ast::call_function(plibbashWalker ctx, ctx->compound_command(ctx); } -bash_ast::walker_pointer bash_ast::create_walker(interpreter& walker) +bash_ast::walker_pointer bash_ast::create_walker(interpreter& walker, + antlr_pointer<ANTLR3_COMMON_TREE_NODE_STREAM_struct>& nodes) { set_interpreter(&walker); walker.push_current_ast(this); diff --git a/src/core/bash_ast.h b/src/core/bash_ast.h index 22635a1..190b020 100644 --- a/src/core/bash_ast.h +++ b/src/core/bash_ast.h @@ -58,13 +58,13 @@ class bash_ast: public boost::noncopyable antlr_pointer<ANTLR3_COMMON_TOKEN_STREAM_struct> token_stream; antlr_pointer<libbashParser_Ctx_struct> parser; pANTLR3_BASE_TREE ast; - antlr_pointer<ANTLR3_COMMON_TREE_NODE_STREAM_struct> nodes; std::function<pANTLR3_BASE_TREE(libbashParser_Ctx_struct*)> parse; typedef std::unique_ptr<libbashWalker_Ctx_struct, std::function<void(plibbashWalker)>> walker_pointer; void init_parser(const std::string& script, const std::string& script_path); - walker_pointer create_walker(interpreter& walker); + walker_pointer create_walker(interpreter& walker, + antlr_pointer<ANTLR3_COMMON_TREE_NODE_STREAM_struct>& nodes); public: bash_ast(const std::istream& source, @@ -92,7 +92,9 @@ public: typename std::result_of<Functor(plibbashWalker)>::type interpret_with(interpreter& walker, Functor walk) { - walker_pointer p_tree_parser = create_walker(walker); + antlr_pointer<ANTLR3_COMMON_TREE_NODE_STREAM_struct> nodes( + antlr3CommonTreeNodeStreamNewTree(ast, ANTLR3_SIZE_HINT)); + walker_pointer p_tree_parser = create_walker(walker, nodes); return walk(p_tree_parser.get()); } diff --git a/utils/instruo.cpp b/utils/instruo.cpp index dcd3924..704c98c 100644 --- a/utils/instruo.cpp +++ b/utils/instruo.cpp @@ -53,7 +53,7 @@ #include <paludis/package_database.hh> #include <paludis/metadata_key.hh> -#include "builtins/builtin_exceptions.h" +#include "core/exceptions.h" #include "command_line.h" #include "libbash.h" #include "utils/metadata.h" @@ -65,71 +65,63 @@ using std::endl; void worker(const std::shared_ptr<PackageIDSequence> &ids) { - std::unordered_map<std::string, std::vector<std::string>> variables; - - std::shared_ptr<const PackageID> id; unsigned total(0); CategoryNamePart old_cat("OLDCAT"); - while(!ids->empty()) + #pragma omp parallel { - id = *ids->begin(); - ids->pop_front(); - if (id->name().category() != old_cat) + #pragma omp single nowait + while(!ids->empty()) { - std::cout << "Processing " << stringify(id->name().category()) << "..." << std::endl; - old_cat = id->name().category(); - FSPath(CommandLine::get_instance()->a_output_directory.argument() + "/" + - stringify(id->name().category())).mkdir(0755, {fspmkdo_ok_if_exists}); - ++total; - } - - Context i_context("When generating metadata for ID '" + stringify(*id) + "':"); - - variables.clear(); - variables["PN"].push_back(stringify(id->name().package())); - variables["PV"].push_back(stringify(id->version().remove_revision())); - variables["P"].push_back(stringify(id->name().package()) + "-" + - stringify(id->version().remove_revision())); - variables["PR"].push_back(id->version().revision_only()); - variables["PVR"].push_back(stringify(id->version())); - variables["PF"].push_back(stringify(id->name().package()) + "-" + stringify(id->version())); - variables["CATEGORY"].push_back(stringify(id->name().category())); - std::vector<std::string> functions; + std::shared_ptr<const PackageID> id = *ids->begin(); + ids->pop_front(); + if (id->name().category() != old_cat) + { + std::cout << "Processing " << stringify(id->name().category()) << "..." << std::endl; + old_cat = id->name().category(); + FSPath(CommandLine::get_instance()->a_output_directory.argument() + "/" + + stringify(id->name().category())).mkdir(0755, {fspmkdo_ok_if_exists}); + ++total; + } - std::string ebuild_path(CommandLine::get_instance()->a_repository_directory.argument() + - variables["CATEGORY"][0] + "/" + - variables["PN"][0] + "/" + - variables["PN"][0] + "-" + - variables["PVR"][0] + ".ebuild"); - try - { - libbash::interpret(ebuild_path, "utils/isolated-functions.sh", variables, functions); - } - catch(const libbash::interpreter_exception& e) - { - cerr << "Exception occurred while interpreting " << ebuild_path << ". The error message is:\n" - << e.what() << endl; - continue; - } - catch(const return_exception& e) - { - cerr << "Unhandled return exception in " << ebuild_path << ". The error message is:\n" - << e.what() << endl; - continue; - } - catch (...) - { - cerr << "Unhandled exception in " << ebuild_path << endl; - continue; + #pragma omp task + { + Context i_context("When generating metadata for ID '" + stringify(*id) + "':"); + + std::unordered_map<std::string, std::vector<std::string>> variables; + variables["PN"].push_back(stringify(id->name().package())); + variables["PV"].push_back(stringify(id->version().remove_revision())); + variables["P"].push_back(stringify(id->name().package()) + "-" + + stringify(id->version().remove_revision())); + variables["PR"].push_back(id->version().revision_only()); + variables["PVR"].push_back(stringify(id->version())); + variables["PF"].push_back(stringify(id->name().package()) + "-" + stringify(id->version())); + variables["CATEGORY"].push_back(stringify(id->name().category())); + std::vector<std::string> functions; + + std::string ebuild_path(CommandLine::get_instance()->a_repository_directory.argument() + + variables["CATEGORY"][0] + "/" + + variables["PN"][0] + "/" + + variables["PN"][0] + "-" + + variables["PVR"][0] + ".ebuild"); + try + { + libbash::interpret(ebuild_path, variables, functions); + + std::string output_path(CommandLine::get_instance()->a_output_directory.argument() + "/" + + variables["CATEGORY"][0] + "/" + + variables["PN"][0] + "-" + + variables["PVR"][0]); + FSPath(output_path).dirname().mkdir(0755, {fspmkdo_ok_if_exists}); + std::ofstream output(output_path, std::ofstream::out | std::ofstream::trunc); + write_metadata(output, variables, functions); + } + catch(const libbash::interpreter_exception& e) + { + cerr << "Exception occurred while interpreting " << ebuild_path << ". The error message is:\n" + << e.what() << endl; + } + } } - - std::string output_path(CommandLine::get_instance()->a_output_directory.argument() + "/" + - variables["CATEGORY"][0] + "/" + - variables["PN"][0] + "-" + - variables["PVR"][0]); - FSPath(output_path).dirname().mkdir(0755, {fspmkdo_ok_if_exists}); - std::ofstream output(output_path, std::ofstream::out | std::ofstream::trunc); - write_metadata(output, variables, functions); } } |