#!/usr/bin/python -b # Copyright 1999-2017 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # # dispatch-conf -- Integrate modified configs, post-emerge # # Jeremy Wohl (http://igmus.org) # # TODO # dialog menus # from __future__ import print_function, unicode_literals import atexit import io import re import subprocess import sys from stat import ST_GID, ST_MODE, ST_UID from random import random from os import path as osp if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "lib")) import portage portage._internal_caller = True from portage import os, shutil from portage import _encodings, _unicode_decode from portage.dispatch_conf import diffstatusoutput, diff_mixed_wrapper from portage.process import find_binary, spawn FIND_EXTANT_CONFIGS = "find '%s' %s -name '._cfg????_%s' ! -name '.*~' ! -iname '.*.bak' -print" DIFF_CONTENTS = "diff -Nu '%s' '%s'" if "case-insensitive-fs" in portage.settings.features: FIND_EXTANT_CONFIGS = FIND_EXTANT_CONFIGS.replace( "-name '._cfg", "-iname '._cfg") # We need a secure scratch dir and python does silly verbose errors on the use of tempnam oldmask = os.umask(0o077) SCRATCH_DIR = None while SCRATCH_DIR is None: try: mydir = "/tmp/dispatch-conf." for x in range(0,8): if int(random() * 3) == 0: mydir += chr(int(65+random()*26.0)) elif int(random() * 2) == 0: mydir += chr(int(97+random()*26.0)) else: mydir += chr(int(48+random()*10.0)) if os.path.exists(mydir): continue os.mkdir(mydir) SCRATCH_DIR = mydir except OSError as e: if e.errno != 17: raise os.umask(oldmask) # Ensure the scratch dir is deleted def cleanup(mydir=SCRATCH_DIR): shutil.rmtree(mydir) atexit.register(cleanup) MANDATORY_OPTS = [ 'archive-dir', 'diff', 'replace-cvs', 'replace-wscomments', 'merge' ] def cmd_var_is_valid(cmd): """ Return true if the first whitespace-separated token contained in cmd is an executable file, false otherwise. """ cmd = portage.util.shlex_split(cmd) if not cmd: return False if os.path.isabs(cmd[0]): return os.access(cmd[0], os.EX_OK) return find_binary(cmd[0]) is not None diff = diff_mixed_wrapper(diffstatusoutput, DIFF_CONTENTS) class dispatch: options = {} def grind (self, config_paths): confs = [] count = 0 config_root = portage.settings["EPREFIX"] or os.sep self.options = portage.dispatch_conf.read_config(MANDATORY_OPTS) if "log-file" in self.options: if os.path.isfile(self.options["log-file"]): shutil.copy(self.options["log-file"], self.options["log-file"] + '.old') if os.path.isfile(self.options["log-file"]) \ or not os.path.exists(self.options["log-file"]): open(self.options["log-file"], 'w').close() # Truncate it os.chmod(self.options["log-file"], 0o600) pager = self.options.get("pager") if pager is None or not cmd_var_is_valid(pager): pager = os.environ.get("PAGER") if pager is None or not cmd_var_is_valid(pager): pager = "cat" pager_basename = os.path.basename(portage.util.shlex_split(pager)[0]) if pager_basename == "less": less_opts = self.options.get("less-opts") if less_opts is not None and less_opts.strip(): pager += " " + less_opts if pager_basename == "cat": pager = "" else: pager = " | " + pager # # Build list of extant configs # for path in config_paths: path = portage.normalize_path( os.path.join(config_root, path.lstrip(os.sep))) # Protect files that don't exist (bug #523684). If the # parent directory doesn't exist, we can safely skip it. if not os.path.isdir(os.path.dirname(path)): continue basename = "*" find_opts = "-name '.*' -type d -prune -o" if not os.path.isdir(path): path, basename = os.path.split(path) find_opts = "-maxdepth 1" try: path_list = _unicode_decode(subprocess.check_output( portage.util.shlex_split(FIND_EXTANT_CONFIGS % (path, find_opts, basename))), errors='strict').splitlines() except subprocess.CalledProcessError: pass else: confs.extend(self.massage(path_list)) if self.options['use-rcs'] == 'yes': for rcs_util in ("rcs", "ci", "co", "rcsmerge"): if not find_binary(rcs_util): print('dispatch-conf: Error finding all RCS utils and " + \ "use-rcs=yes in config; fatal', file=sys.stderr) return False # config file freezing support frozen_files = set(self.options.get("frozen-files", "").split()) auto_zapped = [] protect_obj = portage.util.ConfigProtect( config_root, config_paths, portage.util.shlex_split( portage.settings.get('CONFIG_PROTECT_MASK', '')), case_insensitive=("case-insensitive-fs" in portage.settings.features)) # # Remove new configs identical to current # and # Auto-replace configs a) whose differences are simply CVS interpolations, # or b) whose differences are simply ws or comments, # or c) in paths now unprotected by CONFIG_PROTECT_MASK, # def f (conf): mrgconf = re.sub(r'\._cfg', '._mrg', conf['new']) archive = os.path.join(self.options['archive-dir'], conf['current'].lstrip('/')) if self.options['use-rcs'] == 'yes': mrgfail = portage.dispatch_conf.rcs_archive(archive, conf['current'], conf['new'], mrgconf) else: mrgfail = portage.dispatch_conf.file_archive(archive, conf['current'], conf['new'], mrgconf) if os.path.lexists(archive + '.dist'): unmodified = len(diff(conf['current'], archive + '.dist')[1]) == 0 else: unmodified = 0 if os.path.exists(mrgconf): if mrgfail or len(diff(conf['new'], mrgconf)[1]) == 0: os.unlink(mrgconf) newconf = conf['new'] else: newconf = mrgconf else: newconf = conf['new'] if newconf == mrgconf and \ self.options.get('ignore-previously-merged') != 'yes' and \ os.path.lexists(archive+'.dist') and \ len(diff(archive+'.dist', conf['new'])[1]) == 0: # The current update is identical to the archived .dist # version that has previously been merged. os.unlink(mrgconf) newconf = conf['new'] mystatus, myoutput = diff(conf['current'], newconf) myoutput_len = len(myoutput) same_file = 0 == myoutput_len if mystatus >> 8 == 2: # Binary files differ same_cvs = False same_wsc = False else: # Extract all the normal diff lines (ignore the headers). mylines = re.findall('^[+-][^\n+-].*$', myoutput, re.MULTILINE) # Filter out all the cvs headers cvs_header = re.compile('# [$]Header:') cvs_lines = list(filter(cvs_header.search, mylines)) same_cvs = len(mylines) == len(cvs_lines) # Filter out comments and whitespace-only changes. # Note: be nice to also ignore lines that only differ in whitespace... wsc_lines = [] for x in [r'^[-+]\s*#', r'^[-+]\s*$']: wsc_lines += list(filter(re.compile(x).match, mylines)) same_wsc = len(mylines) == len(wsc_lines) # Do options permit? same_cvs = same_cvs and self.options['replace-cvs'] == 'yes' same_wsc = same_wsc and self.options['replace-wscomments'] == 'yes' unmodified = unmodified and self.options['replace-unmodified'] == 'yes' if same_file: os.unlink (conf ['new']) self.post_process(conf['current']) if os.path.exists(mrgconf): os.unlink(mrgconf) return False elif conf['current'] in frozen_files: """Frozen files are automatically zapped. The new config has already been archived with a .new suffix. When zapped, it is left with the .new suffix (post_process is skipped), since it hasn't been merged into the current config.""" auto_zapped.append(conf['current']) os.unlink(conf['new']) try: os.unlink(mrgconf) except OSError: pass return False elif unmodified or same_cvs or same_wsc or \ not protect_obj.isprotected(conf['current']): self.replace(newconf, conf['current']) self.post_process(conf['current']) if newconf == mrgconf: os.unlink(conf['new']) elif os.path.exists(mrgconf): os.unlink(mrgconf) return False else: return True confs = [x for x in confs if f(x)] # # Interactively process remaining # valid_input = "qhtnmlezu" def diff_pager(file1, file2): cmd = self.options['diff'] % (file1, file2) cmd += pager spawn_shell(cmd) diff_pager = diff_mixed_wrapper(diff_pager) for conf in confs: count = count + 1 newconf = conf['new'] mrgconf = re.sub(r'\._cfg', '._mrg', newconf) if os.path.exists(mrgconf): newconf = mrgconf show_new_diff = 0 while 1: clear_screen() if show_new_diff: diff_pager(conf['new'], mrgconf) show_new_diff = 0 else: diff_pager(conf['current'], newconf) print() print('>> (%i of %i) -- %s' % (count, len(confs), conf ['current'])) print('>> q quit, h help, n next, e edit-new, z zap-new, u use-new\n m merge, t toggle-merge, l look-merge: ', end=' ') # In some cases getch() will return some spurious characters # that do not represent valid input. If we don't validate the # input then the spurious characters can cause us to jump # back into the above "diff" command immediatly after the user # has exited it (which can be quite confusing and gives an # "out of control" feeling). while True: c = getch() if c in valid_input: sys.stdout.write('\n') sys.stdout.flush() break if c == 'q': sys.exit (0) if c == 'h': self.do_help () continue elif c == 't': if newconf == mrgconf: newconf = conf['new'] elif os.path.exists(mrgconf): newconf = mrgconf continue elif c == 'n': break elif c == 'm': merged = SCRATCH_DIR+"/"+os.path.basename(conf['current']) print() ret = os.system (self.options['merge'] % (merged, conf ['current'], newconf)) ret = os.WEXITSTATUS(ret) if ret < 2: ret = 0 if ret: print("Failure running 'merge' command") continue shutil.copyfile(merged, mrgconf) os.remove(merged) mystat = os.lstat(conf['new']) os.chmod(mrgconf, mystat[ST_MODE]) os.chown(mrgconf, mystat[ST_UID], mystat[ST_GID]) newconf = mrgconf continue elif c == 'l': show_new_diff = 1 continue elif c == 'e': if 'EDITOR' not in os.environ: os.environ['EDITOR']='nano' os.system(os.environ['EDITOR'] + ' ' + newconf) continue elif c == 'z': os.unlink(conf['new']) if os.path.exists(mrgconf): os.unlink(mrgconf) break elif c == 'u': self.replace(newconf, conf ['current']) self.post_process(conf['current']) if newconf == mrgconf: os.unlink(conf['new']) elif os.path.exists(mrgconf): os.unlink(mrgconf) break else: raise AssertionError("Invalid Input: %s" % c) if auto_zapped: print() print(" One or more updates are frozen and have been automatically zapped:") print() for frozen in auto_zapped: print(" * '%s'" % frozen) print() def replace (self, newconf, curconf): """Replace current config with the new/merged version. Also logs the diff of what changed into the configured log file.""" if "log-file" in self.options: status, output = diff(curconf, newconf) with io.open(self.options["log-file"], mode="a", encoding=_encodings["stdio"]) as f: f.write(output + "\n") try: os.rename(newconf, curconf) except (IOError, os.error) as why: print('dispatch-conf: Error renaming %s to %s: %s; fatal' % \ (newconf, curconf, str(why)), file=sys.stderr) def post_process(self, curconf): archive = os.path.join(self.options['archive-dir'], curconf.lstrip('/')) if self.options['use-rcs'] == 'yes': portage.dispatch_conf.rcs_archive_post_process(archive) else: portage.dispatch_conf.file_archive_post_process(archive) def massage (self, newconfigs): """Sort, rstrip, remove old versions, break into triad hash. Triad is dictionary of current (/etc/make.conf), new (/etc/._cfg0003_make.conf) and dir (/etc). We keep ._cfg0002_conf over ._cfg0001_conf and ._cfg0000_conf. """ h = {} configs = [] newconfigs.sort () for nconf in newconfigs: # Use strict mode here, because we want to know if it fails, # and portage only merges files with valid UTF-8 encoding. nconf = _unicode_decode(nconf, errors='strict').rstrip() conf = re.sub (r'\._cfg\d+_', '', nconf) dirname = os.path.dirname(nconf) conf_map = { 'current' : conf, 'dir' : dirname, 'new' : nconf, } if conf in h: mrgconf = re.sub(r'\._cfg', '._mrg', h[conf]['new']) if os.path.exists(mrgconf): os.unlink(mrgconf) os.unlink(h[conf]['new']) h[conf].update(conf_map) else: h[conf] = conf_map configs.append(conf_map) return configs def do_help (self): print() print() print(' u -- update current config with new config and continue') print(' z -- zap (delete) new config and continue') print(' n -- skip to next config, leave all intact') print(' e -- edit new config') print(' m -- interactively merge current and new configs') print(' l -- look at diff between pre-merged and merged configs') print(' t -- toggle new config between merged and pre-merged state') print(' h -- this screen') print(' q -- quit') print(); print('press any key to return to diff...', end=' ') getch () def getch (): # from ASPN - Danny Yoo # import tty, termios fd = sys.stdin.fileno() old_settings = termios.tcgetattr(fd) try: tty.setraw(sys.stdin.fileno()) ch = sys.stdin.read(1) finally: termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) return ch def clear_screen(): try: import curses try: curses.setupterm() sys.stdout.write(_unicode_decode(curses.tigetstr("clear"))) sys.stdout.flush() return except curses.error: pass except ImportError: pass os.system("clear 2>/dev/null") shell = os.environ.get("SHELL") if not shell or not os.access(shell, os.EX_OK): shell = find_binary("sh") def spawn_shell(cmd): if shell: sys.__stdout__.flush() sys.__stderr__.flush() spawn([shell, "-c", cmd], env=os.environ, fd_pipes = { 0 : portage._get_stdin().fileno(), 1 : sys.__stdout__.fileno(), 2 : sys.__stderr__.fileno()}) else: os.system(cmd) def usage(argv): print('dispatch-conf: sane configuration file update\n') print('Usage: dispatch-conf [config dirs]\n') print('See the dispatch-conf(1) man page for more details') sys.exit(os.EX_OK) for x in sys.argv: if x in ('-h', '--help'): usage(sys.argv) elif x in ('--version'): print("Portage", portage.VERSION) sys.exit(os.EX_OK) # run d = dispatch () if len(sys.argv) > 1: # for testing d.grind(sys.argv[1:]) else: d.grind(portage.util.shlex_split( portage.settings.get('CONFIG_PROTECT', '')))