Twrapper calls for emerge as soon as new package can be installed. No need to wait for the whole list to be downloaded.

2 files changed, 145 insertions, 275 deletions

diff --git a/twrapper/premerge.sh b/twrapper/premerge.sh
index ca65ded..1c21d0c 100755
--- a/twrapper/premerge.sh
+++ b/twrapper/premerge.sh
@@ -2,7 +2,7 @@
 if [[ -n "${1}" ]]; then
     emerge -f $@
     ./twrapper.py
-    emerge $@
+#    emerge $@
 else
     echo
     echo "Usage:"
diff --git a/twrapper/twrapper.py b/twrapper/twrapper.py
index 9cd3d2a..bd3bd2f 100755
--- a/twrapper/twrapper.py
+++ b/twrapper/twrapper.py
@@ -16,17 +16,19 @@ import signal
 from portage.checksum import perform_md5, verify_all
 from portage.process import spawn
 import idfetch_settings
-import pickle
+#import pickle
 import curses
 import time
 import subprocess
-import shlex
+#import shlex
+import json
 from threading import Thread
 CLEAN_LINE="                                                                                                                                             "
 MAX_ACTIVE_DOWNLOADS=8
 TASK_SPACE=3
 USE_CURSES_FLAG=1
 DEBUG_ON=0
+INSTALL_ON=1
 
 def fit_string(text):
 	text=text+CLEAN_LINE
@@ -61,10 +63,22 @@ def download_msg(index,msg_text):
 def status_msg(index,msg_text):
 	msg(TASK_SPACE*index+1,"   ["+msg_text+"]")
 
-def total_msg(finished_downloads,index,downloaded_size,total_size):
-	msg(2,"Downloaded "+str(finished_downloads)+" of "+str(index)+" distfiles ("\
-					+str(downloaded_size)+" of "+str(total_size)+" bytes = "\
-					+str(downloaded_size*100/total_size)+"% )")
+def total_msg():
+	msg(2,"-> Pkgs: "\
+		+"I:"+str(pkgs_installed_count)\
+		+"/D:"+str(pkgs_downloaded_count)\
+		+"/T:"+str(pkgs_count)\
+		+" - Distfiles D:"+str(distfiles_downloaded_count)\
+		+"/T:"+str(distfiles_count)\
+		+" [Bytes D:"+str(downloaded_size)\
+		+" of T:"+str(total_size)\
+		+" bytes = "+str(downloaded_size*100/total_size)+"%] <-")
+	if install_pkgs_is_running:
+		msg(26,"EMERGE: [INSTALLING...]")
+	else:
+		msg(26,"EMERGE: [IDLE]")
+
+
 
 def progress_msg(index,msg_text1):
 		msg(TASK_SPACE*index+1,msg_text1)
@@ -72,6 +86,9 @@ def progress_msg(index,msg_text1):
 def error_msg(msg_text):
 	msg(max_y-4,"WARNING: "+msg_text)
 
+def emerge_msg(msg_text):
+	msg(max_y-7,"Emerging: "+msg_text)
+
 def open_task_list():
 	debug(" function open_task_list start")
 	idfetch_pkg_list_file_name=idfetch_settings.TASK_DIR+"/pkg_list.list"
@@ -85,20 +102,21 @@ def open_task_list():
 		error_msg("open_task_list(): Can't open file: "+idfetch_pkg_list_file_lock_name)
 	try:
 		idfetch_pkg_list_file = open(idfetch_pkg_list_file_name,"r")
-		idfetch_pkg_list=pickle.load(idfetch_pkg_list_file)
+		idfetch_pkg_list=json.loads(idfetch_pkg_list_file.read())
 		idfetch_pkg_list_file.close()
 	except:
 		idfetch_pkg_list=[]
 		error_msg("open_task_list(): Can't open file: "+idfetch_pkg_list_file_name+"   >>> Nothing to do.")
 		time.sleep(3)
 
-	#delete task list file
-	try:
-		os.unlink(idfetch_pkg_list_file_name)
-		debug("deleted:"+idfetch_pkg_list_file_name)
-	except:
-		error_msg("open_task_list(): Can't delete file: "+idfetch_pkg_list_file_name)
-		time.sleep(3)
+	if not(DEBUG_ON):
+		#delete task list file
+		try:
+#			os.unlink(idfetch_pkg_list_file_name)
+			debug("deleted:"+idfetch_pkg_list_file_name)
+		except:
+			error_msg("open_task_list(): Can't delete file: "+idfetch_pkg_list_file_name)
+			time.sleep(3)
 
 	#unlock file
 	os.unlink(idfetch_pkg_list_file_lock_name)
@@ -107,12 +125,56 @@ def open_task_list():
 
 	return idfetch_pkg_list
 
+class install_pkgs(Thread):
+	def __init__ (self,idfetch_pkg_list):
+		Thread.__init__(self)
+		self.idfetch_pkg_list=idfetch_pkg_list
+		self.status = 2
+	def run(self):
+		global pkgs_installed_count
+		global install_pkgs_is_running
+		while pkgs_installed_count < pkgs_count:
+				self.atom_name=self.idfetch_pkg_list[pkgs_installed_count]['pkg_name'][0:self.idfetch_pkg_list[pkgs_installed_count]['pkg_name'].find("-")]
+				
+				if self.idfetch_pkg_list[pkgs_installed_count]['fetched_distfiles_count'] >= self.idfetch_pkg_list[pkgs_installed_count]['distfiles_count']:
+					self.emerge_subprocess = subprocess.Popen([\
+#						'echo',
+						'/usr/bin/emerge',\
+						self.atom_name],\
+						bufsize=1024,stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
+					mypids.append(self.emerge_subprocess.pid)
+					while 1:
+						try:
+							self.line=self.emerge_subprocess.stdout.readline()
+							if self.line:
+								emerge_msg("pkg "+str(pkgs_installed_count)+" :"+self.atom_name+"   "+self.line.rstrip())
+#										self.igot = re.findall('aksldjfklsjd;lfkj',self.line)
+#										if self.igot:
+#											self.status=0
+#											self.emerge_subprocess.wait()
+#											break
+							else:
+								self.emerge_subprocess.stdout.close()
+								pkgs_installed_count+=1
+								emerge_msg("emerge ended")
+								total_msg()
+								break
+						except:
+							pass
+							error_msg("ERROR!!! CODE:108")
+							self.emerge_subprocess.stdout.close()
+							break
+				else:
+					break
+		install_pkgs_is_running=0
+
 class fetchit(Thread):
-	def __init__ (self,distfile,index):
+	def __init__ (self,parent_pkg, distfile, distfiles_count):
 		Thread.__init__(self)
-#		sms("_init_ in fetchit")
+		debug("_init_ in fetchit")
+		self.parent_pkg=parent_pkg
 		self.distfile = distfile
-		self.index=index
+		self.index=distfiles_count
 		self.status = 2
 		self.place_in_the_list = 1
 #		self.wget_log_file_name = idfetch_settings.TASK_DIR+'/logs/log_wget/'+self.download_file+".log"
@@ -187,191 +249,15 @@ class fetchit(Thread):
 	def run(self):
 		sms("started "+str(self.index))
 		global exit_flag
-#		self.download_file_url_list_file=open(idfetch_settings.TASK_DIR+'/urls/'+self.download_file+".urllist")
-#		self.download_file_url_list=pickle.load(self.download_file_url_list_file)
-#		self.download_file_url_list_file.close()
 		self.myfile_path=idfetch_settings.DIST_DIR+"/"+self.distfile['name']
-#	for myfile in filedict:
 #		"""
 #		fetched  status
 #		0        nonexistent
 #		1        partially downloaded
 #		2        completely downloaded
 #		"""
-#		fetched = 0
-
-#		orig_digests = mydigests.get(myfile, {})
-#		size = orig_digests.get("size")
-
-#		if size == 0:
-			# Zero-byte distfiles are always invalid, so discard their digests.
-#			del mydigests[myfile]
-#			orig_digests.clear()
-#			size = None
-#		pruned_digests = orig_digests
-#		if parallel_fetchonly:
-#			pruned_digests = {}
-#			if size is not None:
-#				pruned_digests["size"] = size
-
-#		myfile_path = os.path.join(mysettings["DISTDIR"], myfile)
-#		has_space = True
-#		has_space_superuser = True
-#		file_lock = None
-#		if listonly:
-#			writemsg_stdout("\n", noiselevel=-1)
-#		else:
-#			# check if there is enough space in DISTDIR to completely store myfile
-			# overestimate the filesize so we aren't bitten by FS overhead
-#			if size is not None and hasattr(os, "statvfs"):
-#				vfs_stat = os.statvfs(mysettings["DISTDIR"])
-#				try:
-#					mysize = os.stat(myfile_path).st_size
-#				except OSError as e:
-#					if e.errno not in (errno.ENOENT, errno.ESTALE):
-#						raise
-#					del e
-#					mysize = 0
-#				if (size - mysize + vfs_stat.f_bsize) >= \
-#					(vfs_stat.f_bsize * vfs_stat.f_bavail):
-
-#					if (size - mysize + vfs_stat.f_bsize) >= \
-#						(vfs_stat.f_bsize * vfs_stat.f_bfree):
-#						has_space_superuser = False
-
-#					if not has_space_superuser:
-#						has_space = False
-#					elif secpass < 2:
-#						has_space = False
-#					elif userfetch:
-#						has_space = False
-
-#			if not has_space:
-#				writemsg(_("!!! Insufficient space to store %s in %s\n") % \
-#					(myfile, mysettings["DISTDIR"]), noiselevel=-1)
-
-#				if has_space_superuser:
-#					writemsg(_("!!! Insufficient privileges to use "
-#						"remaining space.\n"), noiselevel=-1)
-#					if userfetch:
-#						writemsg(_("!!! You may set FEATURES=\"-userfetch\""
-#							" in /etc/make.conf in order to fetch with\n"
-#							"!!! superuser privileges.\n"), noiselevel=-1)
-
-#			if distdir_writable and use_locks:
-
-#				lock_kwargs = {}
-#				if fetchonly:
-#					lock_kwargs["flags"] = os.O_NONBLOCK
-
-#				try:
-#					file_lock = lockfile(myfile_path,
-#						wantnewlockfile=1, **lock_kwargs)
-#				except TryAgain:
-#					writemsg(_(">>> File '%s' is already locked by "
-#						"another fetcher. Continuing...\n") % myfile,
-#						noiselevel=-1)
-#					continue
-#		try:
-#			if not listonly:
-#
-#				eout = EOutput()
-#				eout.quiet = mysettings.get("PORTAGE_QUIET") == "1"
-#				match, mystat = _check_distfile(
-#					myfile_path, pruned_digests, eout)
-#				if match:
-#					if distdir_writable:
-#						try:
-#							apply_secpass_permissions(myfile_path,
-#								gid=portage_gid, mode=0o664, mask=0o2,
-#								stat_cached=mystat)
-#						except PortageException as e:
-#							if not os.access(myfile_path, os.R_OK):
-#								writemsg(_("!!! Failed to adjust permissions:"
-#									" %s\n") % str(e), noiselevel=-1)
-#							del e
-#					continue
-
-#				if distdir_writable and mystat is None:
-#					# Remove broken symlinks if necessary.
-#					try:
-#						os.unlink(myfile_path)
-#					except OSError:
-#						pass
-
-#				if mystat is not None:
-#					if stat.S_ISDIR(mystat.st_mode):
-#						writemsg_level(
-#							_("!!! Unable to fetch file since "
-#							"a directory is in the way: \n"
-#							"!!!   %s\n") % myfile_path,
-#							level=logging.ERROR, noiselevel=-1)
-#						return 0
-
-#					if mystat.st_size == 0:
-#						if distdir_writable:
-#							try:
-#								os.unlink(myfile_path)
-#							except OSError:
-#								pass
-#					elif distdir_writable:
-#						if mystat.st_size < fetch_resume_size and \
-#							mystat.st_size < size:
-#							# If the file already exists and the size does not
-#							# match the existing digests, it may be that the
-#							# user is attempting to update the digest. In this
-#							# case, the digestgen() function will advise the
-#							# user to use `ebuild --force foo.ebuild manifest`
-#							# in order to force the old digests to be replaced.
-#							# Since the user may want to keep this file, rename
-#							# it instead of deleting it.
-#							writemsg(_(">>> Renaming distfile with size "
-#								"%d (smaller than " "PORTAGE_FETCH_RESU"
-#								"ME_MIN_SIZE)\n") % mystat.st_size)
-#							temp_filename = \
-#								_checksum_failure_temp_file(
-#								mysettings["DISTDIR"], myfile)
-#							writemsg_stdout(_("Refetching... "
-#								"File renamed to '%s'\n\n") % \
-#								temp_filename, noiselevel=-1)
-#						elif mystat.st_size >= size:
-#							temp_filename = \
-#								_checksum_failure_temp_file(
-#								mysettings["DISTDIR"], myfile)
-#							writemsg_stdout(_("Refetching... "
-#								"File renamed to '%s'\n\n") % \
-#								temp_filename, noiselevel=-1)
-
-#				if distdir_writable and ro_distdirs:
-#					readonly_file = None
-#					for x in ro_distdirs:
-#						filename = os.path.join(x, myfile)
-#						match, mystat = _check_distfile(
-#							filename, pruned_digests, eout)
-#						if match:
-#							readonly_file = filename
-#							break
-#					if readonly_file is not None:
-#						try:
-#							os.unlink(myfile_path)
-#						except OSError as e:
-#							if e.errno not in (errno.ENOENT, errno.ESTALE):
-#								raise
-#							del e
-#						os.symlink(readonly_file, myfile_path)
-#						continue
-
-#				if fsmirrors and not os.path.exists(myfile_path) and has_space:
-#					for mydir in fsmirrors:
-#						mirror_file = os.path.join(mydir, myfile)
-#						try:
-#							shutil.copyfile(mirror_file, myfile_path)
-#							writemsg(_("Local mirror has file: %s\n") % myfile)
-#							break
-#						except (IOError, OSError) as e:
-#							if e.errno not in (errno.ENOENT, errno.ESTALE):
-#								raise
-#							del e
+		self.fetched = 0
+
 		if 1==1:
 				try:
 					self.mystat = os.stat(self.myfile_path)
@@ -382,42 +268,20 @@ class fetchit(Thread):
 					del e
 					self.fetch()
 				else:
-#					try:
-#						apply_secpass_permissions(
-#							self.myfile_path, gid=portage_gid, mode=0o664, mask=0o2,
-#							stat_cached=self.mystat)
-#					except PortageException as e:
-#						if not os.access(self.myfile_path, os.R_OK):
-#							writemsg(_("!!! Failed to adjust permissions:"
-#								" %s\n") % str(e), noiselevel=-1)
-					if 1==1:
 					# If the file is empty then it's obviously invalid. Remove
 					# the empty file and try to download if possible.
-#					if mystat.st_size == 0:
-#						if distdir_writable:
-#							try:
-#								os.unlink(myfile_path)
-#							except EnvironmentError:
-#								pass
-#					elif myfile not in mydigests:
-						# We don't have a digest, but the file exists.  We must
-						# assume that it is fully downloaded.
-#						continue
-#					else:
+					if self.mystat.st_size == 0:
+							try:
+								os.unlink(myfile_path)
+								self.fetch()
+							except EnvironmentError:
+								debug("Can\'t delete the file")
+					else:
 						if self.mystat.st_size < self.distfile["size"]:
 #							and \
 #							not restrict_fetch:
 							self.fetched = 1 # Try to resume this download.
-#						elif parallel_fetchonly and \
-#							mystat.st_size == mydigests[myfile]["size"]:
-#							eout = EOutput()
-#							eout.quiet = \
-#								mysettings.get("PORTAGE_QUIET") == "1"
-#							eout.ebegin(
-#								"%s size ;-)" % (myfile, ))
-#							eout.eend(0)
-#							continue
-							debug("size too small, continue fetch")
+							debug("Size is small, continue fetching...")
 							self.fetch()
 						else:
 							self.verified_ok, self.reason = verify_all(
@@ -427,17 +291,7 @@ class fetchit(Thread):
 								os.unlink(self.myfile_path)
 								self.fetch()
 							else:
-#								eout = EOutput()
-#								eout.quiet = \
-#									mysettings.get("PORTAGE_QUIET", None) == "1"
-#								digests = mydigests.get(myfile)
-#								if digests:
-#									digests = list(digests)
-#									digests.sort()
-#									eout.ebegin(
-#										"%s %s ;-)" % (myfile, " ".join(digests)))
-#									eout.eend(0)
-#								continue # fetch any remaining files
+								# Already fetched and with correct hashes
 								self.status=0
 
 
@@ -448,7 +302,6 @@ def do_tasks(idfetch_pkg_list):
 	msg(1,"TASK DIR: "+idfetch_settings.TASK_DIR)
 	msg(0,"DOWNLOADING with twrapper...")
 
-
 #	Tidfetch_pkg_list = list of Tidfetch_pkg;
 
 #	Tidfetch_pkg = dict
@@ -459,26 +312,38 @@ def do_tasks(idfetch_pkg_list):
 #		['name'] of string;
 #		['url_list'] of list of strings;
 #		['size'] of int;
-#		['hash  see fetch.py for more details
-#		['hash2
-#		['hash3
+#		['RMD160']  see fetch.py for more details
+#		['SHA1']
+#		['SHA256']
 
 	fetch_distfile_thread_list = []
-	index=0
+	global pkgs_count
+	global pkgs_downloaded_count
+	global pkgs_installed_count
+	global distfiles_count
+	global distfiles_downloaded_count
 	global total_size
-	total_size=0
 	global downloaded_size
+	global install_pkgs_is_running
+	
+	pkgs_count=len(idfetch_pkg_list)
+	pkgs_downloaded_count=0
+	pkgs_installed_count=0
+	distfiles_count=0
+	distfiles_downloaded_count=0
+	total_size=0
 	downloaded_size=0
-#	print(idfetch_pkg_list)
-	while idfetch_pkg_list:
-		current_pkg=idfetch_pkg_list.pop()
-#		print(current_pkg['pkg_name'])
-#		distfiles_list_file = open(idfetch_settings.TASK_DIR+'/pkgs/'+current_pkg)
-#		distfiles_list=pickle.load(distfiles_list_file)
-#		distfiles_list_file.close()
+	install_pkgs_is_running=0
+
+#	idfetch_pkg_list.reverse()
+	prev_pkg=0
+	for current_pkg in idfetch_pkg_list:
+		current_pkg['installed']=0
+		current_pkg['prev_pkg']=prev_pkg
+		current_pkg['distfiles_count']=0
+		current_pkg['fetched_distfiles_count']=0
+		
 		for current_distfile in current_pkg['distfile_list']:
-#			print("distfile",current_distfile['name'])
-
 			#some files may have duplicates, let's skip them
 			no_duplicate=1
 			for cur_fetch_distfile_thread in fetch_distfile_thread_list:
@@ -487,16 +352,19 @@ def do_tasks(idfetch_pkg_list):
 					break
 			if no_duplicate:
 				#create new thread because there is no duplicate
-				index+=1
+#				pkg_contains_distfiles_for_download=1
+				distfiles_count+=1
+				current_pkg['distfiles_count']+=1
 				debug("============================================")
 				debug("pkg:  "+current_pkg['pkg_name'])
 				debug("name: "+current_distfile['name'])
 				debug("size: "+str(current_distfile['size']/1000))
 				total_size+=current_distfile['size']
-				current_fetch_distfile_thread = fetchit(current_distfile, index)
+				current_fetch_distfile_thread = fetchit(current_pkg, current_distfile, distfiles_count)
 				fetch_distfile_thread_list.append(current_fetch_distfile_thread)
+		prev_pkg=current_pkg
 
-	finished_downloads=0
+#	pkgs_downloaded_count
 	place_in_the_list=0
 	running_fetch_distfile_thread_list=[]
 	fetch_distfile_thread_list.reverse()
@@ -509,7 +377,7 @@ def do_tasks(idfetch_pkg_list):
 		if place_in_the_list>=MAX_ACTIVE_DOWNLOADS:
 			break
 
-	total_msg(finished_downloads,index,downloaded_size,total_size)
+	total_msg()
 
 	key='x'
 	if USE_CURSES_FLAG:
@@ -518,15 +386,29 @@ def do_tasks(idfetch_pkg_list):
 	
 		for current_fetch_distfile_thread in running_fetch_distfile_thread_list:
 			if current_fetch_distfile_thread.status==0:
-				finished_downloads+=1
+				distfiles_downloaded_count+=1
 				downloaded_size+=current_fetch_distfile_thread.distfile['size']
-				total_msg(finished_downloads,index,downloaded_size,total_size)
 				running_fetch_distfile_thread_list.remove(current_fetch_distfile_thread)
 				if to_start_fetch_distfile_thread_list:
 				#start next one on the same place in the list
 					starting_fetch_distfile_thread=to_start_fetch_distfile_thread_list.pop()
 					running_fetch_distfile_thread_list.append(starting_fetch_distfile_thread)
 					starting_fetch_distfile_thread.start(current_fetch_distfile_thread.get_place_in_the_list())
+				#install if package has all the files and previous pkgs are installed
+				current_fetch_distfile_thread.parent_pkg['fetched_distfiles_count']+=1
+				if current_fetch_distfile_thread.parent_pkg['fetched_distfiles_count']>=current_fetch_distfile_thread.parent_pkg['distfiles_count']:
+					pkgs_downloaded_count+=1
+					if not(install_pkgs_is_running):
+						if idfetch_pkg_list[pkgs_installed_count]['fetched_distfiles_count'] >= idfetch_pkg_list[pkgs_installed_count]['distfiles_count']:
+							install_pkgs_is_running=1
+							cur_install_pkg_thread=install_pkgs(idfetch_pkg_list)
+							cur_install_pkg_thread.start()
+						else:
+							msg(28,"Next package to install:"+str(pkgs_installed_count+1)\
+								+" pkg_name: "+idfetch_pkg_list[pkgs_installed_count]['pkg_name']\
+								+" fetched:"+str(idfetch_pkg_list[pkgs_installed_count]['fetched_distfiles_count'])\
+								+" of "+str(idfetch_pkg_list[pkgs_installed_count]['distfiles_count']))
+				total_msg()
 			time_msg()
 		# Don't make it too hard for your cpu. Not much changes in 0.5 secs. Let's wait and give cpu some rest.
 		time.sleep(0.5)
@@ -534,8 +416,8 @@ def do_tasks(idfetch_pkg_list):
 			key = stdscr.getch()
 			if (key == ord('q')) or (key == ord('Q')):
 				msg(28,">>>>>>>>>>>>>>>>> EXITING  <<<<<<<<<<<<<<<<<<<<<<<<<<")
-		if finished_downloads == index:
-			msg(28,"All files have been downloaded. Exiting in 3 secs ...")
+		if (distfiles_downloaded_count == distfiles_count) and not(install_pkgs_is_running):
+			msg(28,"All files have been downloaded and installed. Exiting in 3 secs ...")
 			time.sleep(3)
 			exit_flag=1
 			break
@@ -543,6 +425,7 @@ def do_tasks(idfetch_pkg_list):
 
 
 def main(idfetch_pkg_list):
+	
 	try:
 		idfetch_pkg_list=open_task_list()
 		do_tasks(idfetch_pkg_list)
@@ -563,22 +446,9 @@ def main(idfetch_pkg_list):
 			curses.endwin()
 	print("twrapper exited")
 
-def cleanup():                                                                                                                                            
-    while spawned_pids:                                                                                                                               
-        pid = spawned_pids.pop()                                                                                                                  
-        try:                                                                                                                                      
-            if os.waitpid(pid, os.WNOHANG) == (0, 0):                                                                                         
-                os.kill(pid, signal.SIGTERM)                                                                                              
-                os.waitpid(pid, 0)                                                                                                        
-        except OSError:                                                                                                                           
-            # This pid has been cleaned up outside                                                                                            
-            # of spawn().                                                                                                                     
-            pass         
-
-
 if __name__ == '__main__':
 	exit_flag=0
-	task_list=[]
+	idfetch_pkg_list=[]
 	# mypids will hold the pids of all processes created.
 	mypids = []
 	if USE_CURSES_FLAG:
@@ -593,6 +463,6 @@ if __name__ == '__main__':
 		max_y=100
 		max_x=100
 	try:
-		main(task_list)
+		main(idfetch_pkg_list)
 	finally:
 		sys.exit()