diff options
30 files changed, 562 insertions, 105 deletions
diff --git a/LiveCD-HOWTO b/LiveCD-HOWTO index 28872d9..6966a21 100644 --- a/LiveCD-HOWTO +++ b/LiveCD-HOWTO @@ -1,5 +1,6 @@ # Eric Thibodeau -# 01 Jul 2008 +# Initial 01 Jul 2008 +# Updated 17 Aug 2008 # Google Summer of Code 2008: Gentoo Clustering LiveCD ## The build environment ## @@ -19,6 +20,11 @@ If you want to be able to create liveCDs with as little hassle as possible you w # edit /etc/catalyst/catalystrc and add relevant options such as: export MAKEOPTS="-j6" export FEATURES="parallel-fetch" + # You haev to set this here if you want to enable the pbs (torque) use flag for the clustering liveCD + # the server's name is required at compile time (unfortunately) and this is the only way of overriding + # the build host as becoming the one defined as the server. + # This also FORCES the user to use this name as the master node...yerk, I know... + export PBS_SERVER_NAME="master.gentoo.local" # you can set CFLAGS here but it's not recommended # NOTE, don't try to use distcc, it's broken, the distcc user is not being created in the catalyst build @@ -27,11 +33,10 @@ If you want to be able to create liveCDs with as little hassle as possible you w cd /var/svnroot/ svn co svn://anonsvn.gentoo.org/releng - - A recent snapshot (last one used was portage-20080629.tar.bz2), you have to modify the catalyst .spec files accordingly - ** as soon as we get an official 2008.0 release, I will adapt the spect files to that official, nailed down reference ** + - A recent snapshot (last one used was portage-20080806.tar.bz2), you have to modify the catalyst .spec files accordingly So get a snapshot and place it in /var/tmp/catalyst/snapshots (we are doing something evil here...): - wget ftp://mirrors.tera-byte.com/pub/gentoo/snapshots/portage-latest.tar.bz2 -O /var/tmp/catalyst/snapshots/portage-20080629.tar.bz2 + wget ftp://mirrors.tera-byte.com/pub/gentoo/snapshots/portage-20080806.tar.bz2 -O /var/tmp/catalyst/snapshots/portage-20080806.tar.bz2 - A recent stage3, this eliminates many problems related to circular dependancies and collisions. We used drobbin's packages from funtoo.org mkdir -p /var/tmp/catalyst/builds/drobbins/ @@ -45,9 +50,6 @@ If you want to be able to create liveCDs with as little hassle as possible you w git clone git://git.overlays.gentoo.org/proj/clustering-livecd.git - The abovementionned snapshot suffers from the following bugs (whch have been fixed if you take something more recent): - http://bugs.gentoo.org/show_bug.cgi?id=229103 "www-client/mozilla-firefox-bin-2.0.0.14 missing pango-dependency" - http://bugs.gentoo.org/show_bug.cgi?id=206753 "gnome-base/gnome-desktop-2.20.3 - parser errors during emerge" - ** This bug isn't going to be resolved any time soon: http://bugs.gentoo.org/show_bug.cgi?id=142637 "missing dependency in eutils.eclass" @@ -58,7 +60,9 @@ If you want to be able to create liveCDs with as little hassle as possible you w - We also need (some) overlays: # NOTE!!! I set up my layman to put all files into /usr/local/portage/ (and not de thefault of /usr/portage/local) + # If you don't like this, setup the .spec files accrodingly layman -a sunrise + layman -a science ## Before you can build the LiveCD/DVD, you have to build the nfsroot which is to be included in the iso...so skip to the NFSroot-HOWTO and come back here when you're done ;) @@ -70,7 +74,10 @@ If you want to be able to create liveCDs with as little hassle as possible you w ** # Barring the bugs 229103 and 206753, the following sequence should get you down to a LiveCD (haha...HAHA...HAHAHAHAHA) + # These sequences are the ones I put in a script to whipe the slate clean and rebuild the environment + # Well, almost, I don't delete the packages dir except for some packages for which I often want to force a rebuild. + # Stages 1 to 3 cd clustering-livecd/catalyst/releases/2008.0/specs/amd64/ for I in stage* do @@ -79,8 +86,25 @@ If you want to be able to create liveCDs with as little hassle as possible you w # we empty out stuff we don't need anymore ;) umount /var/tmp/catalyst/tmp/default done - mnttmp - catalyst -f livecd-stage1.spec - catalyst -f livecd-stage2.spec + # Minimal CD: + umount /var/git/clustering-livecd/catalyst/livecd/2008.0/overlay/tftproot/nfsroot/x86_64/ + umount /var/tmp/catalyst/tmp/default + + find /var/tmp/catalyst/packages/ -regextype posix-extended -regex '.*(beowulf|aufs|ldap-auth).*' -exec rm -f {} \; + + mount -t tmpfs -o size=7G,noatime catalystbuild /var/tmp/catalyst/tmp/default + schedtool -B -n -4 -e catalyst -f /var/git/clustering-livecd/catalyst/nfsroot/2008.0/specs/amd64/nfsroot_stage4.spec + schedtool -B -n -4 -e catalyst -f /var/git/clustering-livecd/catalyst/livecd/2008.0/specs/amd64/livecd-stage1-minimal.spec + mount -o bind /var/tmp/catalyst/tmp/default/stage4-amd64-nfsroot-20080806/ /var/git/clustering-livecd/catalyst/livecd/2008.0/overlay/tftproot/nfsroot/x86_64/ + + find /var/tmp/catalyst/packages/ -regextype posix-extended -regex '.*(beowulf|aufs|ldap-auth).*' -exec rm -f {} \; + schedtool -B -n -4 -e catalyst -f /var/git/clustering-livecd/catalyst/livecd/2008.0/specs/amd64/livecd-stage2-minimal.spec + + + # "Full" CD: + Same as above but replace livecd-stage?-minimal.spec with livecd-stage?.spec + # At the end, an iso image should be waiting for you as /tmp/livecd-amd64-installer-2008.0.iso ;) + + diff --git a/NFSroot-HOWTO b/NFSroot-HOWTO index 7d09aed..f2445f4 100644 --- a/NFSroot-HOWTO +++ b/NFSroot-HOWTO @@ -1,7 +1,7 @@ # Essentially: # 1- set up the environment as described in the LiveCD-HOWTO # 2- : -# openrc is so haWt we need the latest and gratest...but since we don't want to pull git and all deps into the nfsroot, here is a "hack": +# openrc, we need the latest and gratest...but since we don't want to pull git and all deps into the nfsroot, here is a "hack": cd /var/git/ git clone git://git.overlays.gentoo.org/proj/openrc.git cd openrc/ @@ -21,7 +21,7 @@ done # if you have such a setup ;) # of course...there is also the stage4 in: -# /var/tmp/catalyst/builds/default/stage4-amd64-nfsroot-20080629.tar.bz2 +# /var/tmp/catalyst/builds/default/stage4-amd64-nfsroot-20080806.tar.bz2 # For the liveCD/DVD creation we'll bind mount the folder into the CD/DVD's overlay so it's automatically included in the ISO image: -mount -o bind /var/tmp/catalyst/tmp/default/stage4-amd64-nfsroot-20080629/ /var/git/clustering-livecd/catalyst/livecd/2008.0/overlay/tftproot/nfsroot/x86_64/ +mount -o bind /var/tmp/catalyst/tmp/default/stage4-amd64-nfsroot-20080806/ /var/git/clustering-livecd/catalyst/livecd/2008.0/overlay/tftproot/nfsroot/x86_64/ @@ -0,0 +1,15 @@ +Code: + +cluster-setup + +* Separate the functions so that the node part could be called upon multiple NFSROOTs. This would make it easyer to provide support for different ARCHs provided by the same master. Something in the lines of `setup-nfsroot $NFSROOOT` would be appropriate. + +* Modify script so that it can be used under different ROOT environements (ie: USB environment where the ROOT is actually rw) +- This wasnt done since one can argue that a USBkey install is identical to an HDD install and most of the automation in the cluster-setup are meant for a livecd. HDD and USBkey install should just use the ebuilds and follow the instructions. + +* Modularize so it is easy to add new features to auto-confgure, triggered by USE flags (ie: the torque auto-config is triggered bu the pbs USE flag) + +System: + +* Aufs mount the liveCD and provide the means by which one may save the config+environement to USB key (is it worth the effort or shouldn't someone just install directly to USB). + diff --git a/catalyst/livecd/2008.0/overlay/usr/sbin/cluster-setup b/catalyst/livecd/2008.0/overlay/usr/sbin/cluster-setup index 9675ad8..8d95ca8 100755 --- a/catalyst/livecd/2008.0/overlay/usr/sbin/cluster-setup +++ b/catalyst/livecd/2008.0/overlay/usr/sbin/cluster-setup @@ -4,9 +4,9 @@ # This script is written so parts can be reused for liveCD mounting...eventually ;) . /mnt/livecd/sbin/functions.sh -CDROOT=/mnt/livecd/ -MKDIR=${CDROOT}/bin/mkdir -MODPROBE=${CDROOT}/sbin/modprobe +CONFDIR="/etc/gentoo" +CLUSTER_CONF="$CONFDIR/cluster.conf" +LDAP_CONF="$CONFDIR/ldap_auth.conf" conf_error() { eerror "CONFIG_OK is not set to 'yes' in $1" @@ -19,6 +19,10 @@ conf_error() { } aufs_mount() { + CDROOT=/mnt/livecd/ + MKDIR=${CDROOT}/bin/mkdir + MODPROBE=${CDROOT}/sbin/modprobe + $MODPROBE aufs || eerror "Got $? while probing for aufs module!" # Looping this simply because it should replace the current tmpfs mounting of the liveCD :P for I in tftproot @@ -43,19 +47,56 @@ do_conf() { } # This could have been inlined into do_conf but we would rather check configs _before_ -# we do anything...we could be a little mode selective than /etc/gentoo/* though +# we do anything...we could be a little mode selective than $CONFDIR/* though check_conf() { einfo "Checking config files.." - for I in /etc/gentoo/* + for I in $CONFDIR/*.conf do . $I [[ $CONFIG_OK == "yes" ]] || conf_error $I done } +# This function has to be called after the $CLUSTER_CONF file has been sourced +# Required external vars: +# CLUSTER_CONF:NFSROOT +config_ssh(){ + KEY_NAME="node_dsa" + NETADDR="$(sipcalc ${CLUSTER_ETH} | egrep 'Network address' | cut -d- -f2)" + # The following is needed because $HOME is set to / on the livecd console for some reason + HDIR="/root" + + pushd $PWD + + einfo "Setting up passwordless ssh for root between master and the nodes" + mkdir -p $HDIR/.ssh/ $NFSROOT/root/.ssh/ + cd $HDIR/.ssh/ + ssh-keygen -t dsa -b 1024 -f $KEY_NAME -N "" + cat $KEY_NAME.pub >> $NFSROOT/root/.ssh/authorized_keys + cat >> $HDIR/.ssh/config <<-EOF + # Auto-generated by $0 to ease the use of passwordless logins for all users + # Although this approach is questionnable, it's also one of the only way + # to automate some of the master-node configuration/discovery process + # until an Avahi-style approach is implemented. + Host *.gentoo.local ${NETADDR%.[09]*}.* node* + IdentityFile $HDIR/.ssh/$KEY_NAME + StrictHostKeyChecking no + UserKnownHostsFile /dev/null + CheckHostIP no + Compression no + EOF + + echo "# we don't use IPV6, the following setting removes undue delays that would be added" >> /etc/ssh/ssh_config + echo "AddressFamily inet" >> /etc/ssh/ssh_config + + chmod 600 $HDIR/.ssh/* $NFSROOT/root/.ssh/* + chmod 700 $HDIR/.ssh $NFSROOT/root/.ssh + popd +} + ####### Main ######### -# speed up the dev cycle by automating some tasks: +# Speed up the dev cycle by automating some tasks: if [[ "$1" == "--test" ]]; then ewarn ewarn "Yeah...you know what you're doing right?" @@ -70,17 +111,14 @@ if [[ "$1" == "--test" ]]; then /etc/init.d/net.eth1 restart echo root:test | chpasswd /etc/init.d/sshd start - sed -i -e's:CONFIG_OK="no":CONFIG_OK="yes":' /etc/gentoo/* + sed -i -e's:CONFIG_OK="no":CONFIG_OK="yes":' $CONFDIR/*.conf fi check_conf -aufs_mount -# After this, all paths should be ok (no need to use relative paths with ${CDROOT} - -do_conf /etc/gentoo/ldap_auth.conf "emerge --config =net-nds/ldap-auth-0.1" -/etc/init.d/slapd start || ewarn "Slapd did not start! See above for error messages...continuing" +grep -q /mnt/aufs/tftproot /proc/mounts +[ $? ] && aufs_mount -. /etc/gentoo/cluster.conf +. $CLUSTER_CONF einfo "Making sure $CLUSTER_ETH is configured correctly.." if [[ "$CLUSTER_ETH" =~ eth.* && $1 != "--test" ]]; then echo "The cluster's NIC is set as $CLUSTER_ETH with the following config:" @@ -99,20 +137,37 @@ if [[ "$CLUSTER_ETH" =~ eth.* && $1 != "--test" ]]; then fi [[ $ANS == "n" ]] && einfo "You know what you're doing..." else - einfo "CLUSTER_ETH is not set to an eth? device: $CLUSTER_ETH" + [[ $1 != "--test" ]] && einfo "CLUSTER_ETH is not set to an eth? device: $CLUSTER_ETH" einfo "Hope you know what you're doing." fi -do_conf /etc/gentoo/cluster.conf "emerge --config =sys-cluster/beowulf-head-0.1" +do_conf $CLUSTER_CONF "emerge --config =sys-cluster/beowulf-head-0.1" +config_ssh -# we need to set master and master.$DOMAINNAME to a real IP address for everything to work: +# we need to set master and master.$DOMAINNAME to a real IP address (not 127.0.0.1) for everything to work: echo "$(LC_ALL=C ifconfig "eth1" | sed -n -e 's#.*\(inet addr:\|inet \)\([^ ]*\).*#\2#p') $(hostname).$LOCALDOMAINNAME $(hostname)" >> /etc/hosts -einfo "Setting up auth on the $NFSROOT" -USE=-authmaster ROOT="$NFSROOT" emerge --config =net-nds/ldap-auth-0.1 - einfo "We need to refresh resolv.conf and dnsmasq." einfo "Restarting both $PUBETH and dnsmasq..." /etc/init.d/net.$PUBETH restart /etc/init.d/dnsmasq restart +# This also has to be done with the $CLUSTER_CONF information sourced + +# Setting up Torque +# Checking for pbs USE flag +portageq envvar USE | grep -q pbs + +if [[ $? == 0 ]]; then + einfo "Setting up torque on $HOSTNAME and in $NFSROOT" + emerge --config =sys-cluster/torque-2.3.3 + USE=-server ROOT=$NFSROOT emerge --config =sys-cluster/torque-2.3.3 +fi + +# we want to do this last, otherwise some name resolution problems might haunt you +do_conf $LDAP_CONF "emerge --config =net-nds/ldap-auth-0.1" +/etc/init.d/slapd start || ewarn "Slapd did not start! See above for error messages...continuing" +einfo "Setting up auth on the $NFSROOT" +USE=-authmaster ROOT="$NFSROOT" emerge --config =net-nds/ldap-auth-0.1 + +[[ $? == 0 ]] && einfo "Once you have nodes going, restart Torque (pbs_server)" diff --git a/catalyst/livecd/2008.0/specs/amd64/livecd-stage1-minimal.spec b/catalyst/livecd/2008.0/specs/amd64/livecd-stage1-minimal.spec index f55e05f..8b97eeb 100644 --- a/catalyst/livecd/2008.0/specs/amd64/livecd-stage1-minimal.spec +++ b/catalyst/livecd/2008.0/specs/amd64/livecd-stage1-minimal.spec @@ -3,13 +3,13 @@ version_stamp: minimal-installer-2008.0 target: livecd-stage1 rel_type: default profile: default/linux/amd64/2008.0/desktop -snapshot: 20080806 +snapshot: 20080816 source_subpath: default/stage3-amd64-desktop-2008.0 # Kyron: portage_confdir: /var/git/clustering-livecd/profiles/hpc-clustering/ pkgcache_path: /var/tmp/catalyst/packages/amd64_minimal -portage_overlay: /var/git/clustering-livecd/overlay/ /var/svnroot/livecd-tools +portage_overlay: /var/git/clustering-livecd/overlay/ /var/svnroot/livecd-tools /usr/local/portage/layman/sunrise /usr/local/portage/layman/science ### end # Kyron: added @@ -19,6 +19,7 @@ portage_overlay: /var/git/clustering-livecd/overlay/ /var/svnroot/livecd-tools # -minimal (for openldap to be used as auth server) # -fam to fix circular dependancy preventing the build # nfsexport, nfs, hinotify are for aufs (in stage2) +# pbs, server, cpusets for torque livecd/use: -* deprecated @@ -53,6 +54,9 @@ livecd/use: hinotify nfs nonfs4 + beep + resolvconf + pbs livecd/packages: app-accessibility/brltty @@ -168,3 +172,5 @@ livecd/packages: app-admin/eselect-lapack net-misc/wol sys-apps/dstat + dev-util/strace + dev-util/ltrace diff --git a/catalyst/livecd/2008.0/specs/amd64/livecd-stage1.spec b/catalyst/livecd/2008.0/specs/amd64/livecd-stage1.spec index 3f4a1c9..17ab415 100644 --- a/catalyst/livecd/2008.0/specs/amd64/livecd-stage1.spec +++ b/catalyst/livecd/2008.0/specs/amd64/livecd-stage1.spec @@ -3,13 +3,13 @@ version_stamp: installer-2008.0 target: livecd-stage1 rel_type: default profile: default/linux/amd64/2008.0/desktop -snapshot: 20080806 +snapshot: 20080816 source_subpath: default/stage3-amd64-desktop-2008.0 # Kyron: portage_confdir: /var/git/clustering-livecd/profiles/hpc-clustering/ pkgcache_path: /var/tmp/catalyst/packages/amd64_desktop -portage_overlay: /var/git/clustering-livecd/overlay/ /var/svnroot/livecd-tools +portage_overlay: /var/git/clustering-livecd/overlay/ /var/svnroot/livecd-tools /usr/local/portage/layman/sunrise /usr/local/portage/layman/science ### end # Kyron: added @@ -19,6 +19,7 @@ portage_overlay: /var/git/clustering-livecd/overlay/ /var/svnroot/livecd-tools # -minimal (for openldap to be used as auth server) # -fam to fix circular dependancy preventing the build # nfsexport, nfs, hinotify are for aufs (in stage2) +# tk, server, cpusets for torque livecd/use: branding livecd @@ -34,6 +35,9 @@ livecd/use: hinotify nfs nonfs4 + beep + resolvconf + tk livecd/packages: app-accessibility/brltty @@ -215,4 +219,6 @@ livecd/packages: dev-util/strace sys-process/lsof sys-apps/dstat + dev-util/strace + dev-util/ltrace diff --git a/catalyst/livecd/2008.0/specs/amd64/livecd-stage2-minimal.spec b/catalyst/livecd/2008.0/specs/amd64/livecd-stage2-minimal.spec index 604fe02..0f811a1 100644 --- a/catalyst/livecd/2008.0/specs/amd64/livecd-stage2-minimal.spec +++ b/catalyst/livecd/2008.0/specs/amd64/livecd-stage2-minimal.spec @@ -3,13 +3,13 @@ version_stamp: minimal-installer-2008.0 target: livecd-stage2 rel_type: default profile: default/linux/amd64/2008.0/desktop -snapshot: 20080806 +snapshot: 20080816 source_subpath: default/livecd-stage1-amd64-minimal-installer-2008.0 # Kyron: portage_confdir: /var/git/clustering-livecd/profiles/hpc-clustering/ pkgcache_path: /var/tmp/catalyst/packages/amd64_minimal -portage_overlay: /var/git/clustering-livecd/overlay/ /var/svnroot/livecd-tools /usr/local/portage/layman/sunrise +portage_overlay: /var/git/clustering-livecd/overlay/ /var/svnroot/livecd-tools /usr/local/portage/layman/sunrise /usr/local/portage/layman/science livecd/iso: /tmp/livecd-amd64-minimal-2008.0.iso livecd/volid: x86_64 Gentoo Clustering LiveCD @@ -33,8 +33,7 @@ livecd/root_overlay: /var/git/clustering-livecd/catalyst/livecd/2008.0/overlay/ boot/kernel: gentoo boot/kernel/gentoo/sources: gentoo-sources -boot/kernel/gentoo/config: /var/git/clustering-livecd/catalyst/livecd/2008.0/kconfig/livecd-2.6.24.config -#boot/kernel/gentoo/config: /var/svnroot/releng/trunk/releases/2008.0/kconfig/amd64/installcd-2.6.24.config +boot/kernel/gentoo/config: /var/git/clustering-livecd/catalyst/livecd/2008.0/kconfig/livecd-2.6.25.config boot/kernel/gentoo/use: -* atm @@ -63,6 +62,12 @@ boot/kernel/gentoo/use: nfs hinotify nonfs4 + beep + resolvconf + pbs + server +# Kernel support is built in but this feature is quite experimental in Torque +# cpusets boot/kernel/gentoo/packages: net-dialup/globespan-adsl @@ -72,6 +77,7 @@ boot/kernel/gentoo/packages: sys-fs/ntfs3g ## kyron: let the fun begin, we need a unionfs-ish FS to be able to configure the clustering stuff on the liveCD sys-fs/aufs + sys-cluster/torque livecd/rcadd: ntpd|default diff --git a/catalyst/livecd/2008.0/specs/amd64/livecd-stage2.spec b/catalyst/livecd/2008.0/specs/amd64/livecd-stage2.spec index 0bd5f95..2e0097f 100644 --- a/catalyst/livecd/2008.0/specs/amd64/livecd-stage2.spec +++ b/catalyst/livecd/2008.0/specs/amd64/livecd-stage2.spec @@ -3,13 +3,13 @@ version_stamp: 2008.0 target: livecd-stage2 rel_type: default profile: default/linux/amd64/2008.0/desktop -snapshot: 20080806 +snapshot: 20080816 source_subpath: default/livecd-stage1-amd64-installer-2008.0 # Kyron: portage_confdir: /var/git/clustering-livecd/profiles/hpc-clustering/ pkgcache_path: /var/tmp/catalyst/packages/amd64_desktop -portage_overlay: /var/git/clustering-livecd/overlay/ /var/svnroot/livecd-tools /usr/local/portage/layman/sunrise +portage_overlay: /var/git/clustering-livecd/overlay/ /var/svnroot/livecd-tools /usr/local/portage/layman/sunrise /usr/local/portage/layman/science livecd/iso: /tmp/livecd-amd64-installer-2008.0.iso livecd/volid: x86_64 Gentoo Clustering LiveCD @@ -33,7 +33,7 @@ livecd/root_overlay: /var/git/clustering-livecd/catalyst/livecd/2008.0/overlay/ boot/kernel: gentoo boot/kernel/gentoo/sources: gentoo-sources -boot/kernel/gentoo/config: /var/git/clustering-livecd/catalyst/livecd/2008.0/kconfig/livecd-2.6.24.config +boot/kernel/gentoo/config: /var/git/clustering-livecd/catalyst/livecd/2008.0/kconfig/livecd-2.6.25.config boot/kernel/gentoo/use: atm fbcondecor @@ -45,6 +45,13 @@ boot/kernel/gentoo/use: nfs hinotify nonfs4 + beep + resolvconf + pbs + tk + server +# Kernel support is built in but this feature is quite experimental in Torque +# cpusets boot/kernel/gentoo/packages: media-libs/alsa-oss @@ -56,7 +63,7 @@ boot/kernel/gentoo/packages: sys-fs/ntfs3g ## kyron: let the fun begin, we need a unionfs-ish FS to be able to configure the clustering stuff on the liveCD sys-fs/aufs -# sys-block/nbd + sys-cluster/torque livecd/empty: /var/tmp diff --git a/catalyst/livecd/2008.0/specs/amd64/stage1.spec b/catalyst/livecd/2008.0/specs/amd64/stage1.spec index a5c35e2..a182d44 100644 --- a/catalyst/livecd/2008.0/specs/amd64/stage1.spec +++ b/catalyst/livecd/2008.0/specs/amd64/stage1.spec @@ -3,7 +3,7 @@ target: stage1 version_stamp: 2008.0 rel_type: default profile: default/linux/amd64/2008.0 -snapshot: 20080806 +snapshot: 20080816 source_subpath: drobbins/stage3-amd64-2008.06.27 # Kyron: diff --git a/catalyst/livecd/2008.0/specs/amd64/stage2-desktop.spec b/catalyst/livecd/2008.0/specs/amd64/stage2-desktop.spec index 168beaa..13a8e6e 100644 --- a/catalyst/livecd/2008.0/specs/amd64/stage2-desktop.spec +++ b/catalyst/livecd/2008.0/specs/amd64/stage2-desktop.spec @@ -3,7 +3,7 @@ target: stage2 version_stamp: desktop-2008.0 rel_type: default profile: default/linux/amd64/2008.0/desktop -snapshot: 20080806 +snapshot: 20080816 source_subpath: default/stage1-amd64-2008.0 # Kyron: diff --git a/catalyst/livecd/2008.0/specs/amd64/stage3-desktop.spec b/catalyst/livecd/2008.0/specs/amd64/stage3-desktop.spec index 5a37872..b424eff 100644 --- a/catalyst/livecd/2008.0/specs/amd64/stage3-desktop.spec +++ b/catalyst/livecd/2008.0/specs/amd64/stage3-desktop.spec @@ -3,7 +3,7 @@ target: stage3 version_stamp: desktop-2008.0 rel_type: default profile: default/linux/amd64/2008.0/desktop -snapshot: 20080806 +snapshot: 20080816 source_subpath: default/stage2-amd64-desktop-2008.0 # Kyron: diff --git a/catalyst/nfsroot/2008.0/scripts/nfsroot_setup.sh b/catalyst/nfsroot/2008.0/scripts/nfsroot_setup.sh index 569e721..4f9b7c9 100644 --- a/catalyst/nfsroot/2008.0/scripts/nfsroot_setup.sh +++ b/catalyst/nfsroot/2008.0/scripts/nfsroot_setup.sh @@ -1,18 +1,15 @@ #!/bin/bash - # (C) Eric Thibodeau GPL v2 -. /sbin/functions.sh - -eeinfo(){ - einfo "=====================================" - einfo "$*" - einfo "=====================================" +eecho(){ + echo "=====================================" + echo "$*" + echo "=====================================" } kernel_links() { - eeinfo "Correcting kernel and initramfs links" + eecho "Correcting kernel and initramfs links" cd /boot ln -s kernel-* vmlinuz ln -s initramfs-* initramfs @@ -21,7 +18,7 @@ kernel_links() unionfs_prep() { - eeinfo "=adding unionfs required dirs========" + eecho "=adding unionfs required dirs========" # for I in etc var tmp # do mkdir -p /mnt/rw_mounts/ @@ -54,7 +51,7 @@ change_opt() { } openrc_diskless_setup() { - eeinfo "=Setting up default RC configs=======" + eecho "=Setting up default RC configs=======" # /etc/rc.conf for I in 'rc_parallel="yes"' 'rc_depend_strict="NO"' 'rc_tty_number=2' @@ -75,39 +72,39 @@ openrc_diskless_setup() { # We don't do this anymore since it puts files into /lib64/rc/init.d/ # and that causes error messages on reboot: - #einfo "Pre-generating dependencies..." + #echo "Pre-generating dependencies..." #/lib/rc/bin/rc-depend -u # Instead, we actually clean out that folder: rm -Rf /lib64/rc/init.d/* # The following is only useful if some freak grabs the livecd and is in a TZ # making the /etc files dated in the future (my system is UTC-5 - einfo "Moving dates to the past so the Cache isn't always regenerated" + echo "Moving dates to the past so the Cache isn't always regenerated" find /etc/ -exec touch --date=yesterday {} \; } setup_services() { # Pre-generating sshd keys can be the source of philosophical debates: - eeinfo "=Pre-generating sshd keys============" + eecho "=Pre-generating sshd keys============" . /etc/init.d/sshd gen_keys # ln -s /etc/runlevels/default /etc/runlevels/unionfs } dash_is_sh() { - eeinfo "=Replacing sh with dash==============" + eecho "=Replacing sh with dash==============" rm /bin/sh ln -s /bin/dash /bin/sh } clear_resolv.conf(){ # we don't want the buildhost's resolv.conf to lie around and confuse things - eeinfo "=Clearing our resolv.conf============" + eecho "=Clearing our resolv.conf============" echo "" > /etc/resolv.conf } setup_ntp-client(){ - eeinfo "=Configuring ntp-client==============" + eecho "=Configuring ntp-client==============" sed -ie 's:NTPCLIENT_OPTS=.*:NTPCLIENT_OPTS="-s -b -u master.gentoo.local":' /etc/conf.d/ntp-client } @@ -120,7 +117,7 @@ openrc_diskless_setup setup_services # TEMPORARY for testing: -eeinfo "=Changing root password==============" +eecho "=Changing root password==============" echo root:test | chpasswd exit 0 diff --git a/catalyst/nfsroot/2008.0/specs/amd64/nfsroot_stage4.spec b/catalyst/nfsroot/2008.0/specs/amd64/nfsroot_stage4.spec index 9998844..7567b8d 100644 --- a/catalyst/nfsroot/2008.0/specs/amd64/nfsroot_stage4.spec +++ b/catalyst/nfsroot/2008.0/specs/amd64/nfsroot_stage4.spec @@ -12,7 +12,7 @@ subarch: amd64 # example: # version_stamp: 2006.1 #version_stamp: nfsroot-20080629 -version_stamp: nfsroot-20080806 +version_stamp: nfsroot-20080816 # The target specifies what target we want catalyst to do. For building a CD, # we start with stage4 as our target. @@ -34,7 +34,7 @@ profile: default/linux/amd64/2008.0/ # This specifies which snapshot to use for building this target. # example: -snapshot: 20080806 +snapshot: 20080816 # This specifies where the seed stage comes from for this target, The path is # relative to $clst_sharedir/builds. The rel_type is also used as a path prefix# for the seed. @@ -60,7 +60,7 @@ portage_confdir: /var/git/clustering-livecd/profiles/hpc-clustering/ # have used when building this target. # example: # portage_overlay: /usr/local/portage -portage_overlay: /var/git/clustering-livecd/overlay/ /var/svnroot/livecd-tools /usr/local/portage/layman/sunrise +portage_overlay: /var/git/clustering-livecd/overlay/ /var/svnroot/livecd-tools /usr/local/portage/layman/sunrise /usr/local/portage/layman/science # This allows the optional directory containing the output packages for # catalyst. Mainly used as a way for different spec files to access the same @@ -84,11 +84,14 @@ pkgcache_path: /var/tmp/catalyst/packages/amd64 # Leaving this blank will default to the system use flags # example: # stage4/use: ipv6 socks5 fbcon ncurses readline ssl +# -tk, -server for torque stage4/use: minimal openmp mpi - + pbs + -tk + -server # This is the set of packages that we will merge into the stage4 tarball. They # will be built with the USE flags configured above. These packages must not @@ -114,6 +117,7 @@ stage4/packages: app-admin/eselect-cblas app-admin/eselect-blas app-admin/eselect-lapack + # A fsscript is simply a shell script that is copied into the chroot of the stage4 diff --git a/catalyst/nfsroot/2008.0/specs/amd64/stage1.spec b/catalyst/nfsroot/2008.0/specs/amd64/stage1.spec index a5c35e2..a182d44 100644 --- a/catalyst/nfsroot/2008.0/specs/amd64/stage1.spec +++ b/catalyst/nfsroot/2008.0/specs/amd64/stage1.spec @@ -3,7 +3,7 @@ target: stage1 version_stamp: 2008.0 rel_type: default profile: default/linux/amd64/2008.0 -snapshot: 20080806 +snapshot: 20080816 source_subpath: drobbins/stage3-amd64-2008.06.27 # Kyron: diff --git a/catalyst/nfsroot/2008.0/specs/amd64/stage2.spec b/catalyst/nfsroot/2008.0/specs/amd64/stage2.spec index 9612e94..9aa22b0 100644 --- a/catalyst/nfsroot/2008.0/specs/amd64/stage2.spec +++ b/catalyst/nfsroot/2008.0/specs/amd64/stage2.spec @@ -3,7 +3,7 @@ target: stage2 version_stamp: 2008.0 rel_type: default profile: default/linux/amd64/2008.0 -snapshot: 20080806 +snapshot: 20080816 source_subpath: default/stage1-amd64-2008.0 # Kyron: diff --git a/catalyst/nfsroot/2008.0/specs/amd64/stage3.spec b/catalyst/nfsroot/2008.0/specs/amd64/stage3.spec index 4b86af8..f736964 100644 --- a/catalyst/nfsroot/2008.0/specs/amd64/stage3.spec +++ b/catalyst/nfsroot/2008.0/specs/amd64/stage3.spec @@ -3,7 +3,7 @@ target: stage3 version_stamp: 2008.0 rel_type: default profile: default/linux/amd64/2008.0/ -snapshot: 20080806 +snapshot: 20080816 source_subpath: default/stage2-amd64-2008.0 # Kyron: diff --git a/overlay/sys-apps/openrc/Manifest b/overlay/sys-apps/openrc/Manifest new file mode 100644 index 0000000..9f81d64 --- /dev/null +++ b/overlay/sys-apps/openrc/Manifest @@ -0,0 +1,2 @@ +DIST openrc-0.2.5.tar.bz2 144248 RMD160 33275cea6592abba677acd40857a0ec434eb0787 SHA1 36190b706cc7eb8dd5b612cfcbb4d0175630ecb2 SHA256 bdf89a670cf1fd8b4346b09defa3c38c584b08bec97997f095c4e798530cab80 +EBUILD openrc-0.2.5-r1.ebuild 9023 RMD160 a97fd7819621b6fb738fd013a63d46720aa6cf6f SHA1 1117c31839946029c3c93cbf1dfc2f052ffdc62c SHA256 ef943aad5690c41d5ed017e93ce2acea8f2b03a1aa50a92478da15a4e26ca90e diff --git a/overlay/sys-apps/openrc/openrc-0.2.5-r1.ebuild b/overlay/sys-apps/openrc/openrc-0.2.5-r1.ebuild new file mode 100644 index 0000000..d6988ca --- /dev/null +++ b/overlay/sys-apps/openrc/openrc-0.2.5-r1.ebuild @@ -0,0 +1,276 @@ +# Copyright 1999-2008 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Header: /var/cvsroot/gentoo-x86/sys-apps/openrc/openrc-0.2.5.ebuild,v 1.4 2008/06/09 14:38:45 cardoe Exp $ + +inherit eutils flag-o-matic multilib toolchain-funcs + +if [[ ${PV} == "9999" ]] ; then + EGIT_REPO_URI="git://git.overlays.gentoo.org/proj/openrc.git" + EGIT_BRANCH="master" + inherit git +else + SRC_URI="http://roy.marples.name/${PN}/${P}.tar.bz2 + mirror://gentoo/${P}.tar.bz2 + http://dev.gentoo.org/~cardoe/files/${P}.tar.bz2 + http://dev.gentoo.org/~vapier/dist/${P}.tar.bz2" +fi + +DESCRIPTION="OpenRC manages the services, startup and shutdown of a host" +HOMEPAGE="http://roy.marples.name/openrc" + +LICENSE="BSD-2" +SLOT="0" +KEYWORDS="~alpha ~amd64 ~arm ~hppa ~ia64 ~m68k ~mips ~ppc ~ppc64 ~s390 ~sh ~sparc ~sparc-fbsd ~x86 ~x86-fbsd" +IUSE="debug ncurses pam unicode kernel_linux kernel_FreeBSD" + +RDEPEND="virtual/init + kernel_linux? ( >=sys-apps/module-init-tools-3.2.2-r2 ) + kernel_FreeBSD? ( sys-process/fuser-bsd ) + elibc_glibc? ( >=sys-libs/glibc-2.5 ) + ncurses? ( sys-libs/ncurses ) + pam? ( virtual/pam ) + >=sys-apps/baselayout-2.0.0 + !<sys-fs/udev-118-r2" +DEPEND="${RDEPEND} + virtual/os-headers" + +pkg_setup() { + LIBDIR="lib" + [ "${SYMLINK_LIB}" = "yes" ] && LIBDIR=$(get_abi_LIBDIR "${DEFAULT_ABI}") + + MAKE_ARGS="${MAKE_ARGS} LIBNAME=${LIBDIR}" + + local brand="Unknown" + if use kernel_linux ; then + MAKE_ARGS="${MAKE_ARGS} OS=Linux" + brand="Linux" + elif use kernel_FreeBSD ; then + MAKE_ARGS="${MAKE_ARGS} OS=FreeBSD" + brand="FreeBSD" + fi + export BRANDING="Gentoo ${brand}" + + export DEBUG=$(usev debug) + export MKPAM=$(usev pam) + export MKTERMCAP=$(usev ncurses) +} + +src_unpack() { + if [[ ${PV} == "9999" ]] ; then + git_src_unpack + else + unpack ${A} + fi + cd "${S}" + epatch "${FILESDIR}"/9999/*.patch +} + +src_compile() { + # catch people running `ebuild` w/out setup + if [[ -z ${MAKE_ARGS} ]] ; then + die "Your MAKE_ARGS is empty ... are you running 'ebuild' but forgot to execute 'setup' ?" + fi + + if [[ ${PV} == "9999" ]] ; then + local ver="git-$(git --git-dir=${EGIT_STORE_DIR}/${EGIT_PROJECT} rev-parse --verify ${EGIT_BRANCH} | cut -c1-8)" + sed -i "/^VERSION[[:space:]]*=/s:=.*:=${ver}:" Makefile + fi + + tc-export CC AR RANLIB + echo emake ${MAKE_ARGS} + emake ${MAKE_ARGS} || die "emake ${MAKE_ARGS} failed" +} + +src_install() { + emake ${MAKE_ARGS} DESTDIR="${D}" install || die "make install failed" + gen_usr_ldscript libeinfo.so + gen_usr_ldscript librc.so + + keepdir /"${LIBDIR}"/rc/init.d + keepdir /"${LIBDIR}"/rc/tmp + + # Backup our default runlevels + dodir /usr/share/"${PN}" + mv "${D}/etc/runlevels" "${D}/usr/share/${PN}" + + # Setup unicode defaults for silly unicode users + use unicode && sed -i -e '/^unicode=/s:NO:YES:' "${D}"/etc/rc.conf + + # Cater to the norm + (use x86 || use amd64) && sed -i -e '/^windowkeys=/s:NO:YES:' "${D}"/etc/conf.d/keymaps +} + +add_boot_init() { + local initd=$1 + # if the initscript is not going to be installed and is not + # currently installed, return + [[ -e ${D}/etc/init.d/${initd} || -e ${ROOT}/etc/init.d/${initd} ]] \ + || return + [[ -e ${ROOT}/etc/runlevels/boot/${initd} ]] && return + elog "Auto-adding '${initd}' service to your boot runlevel" + ln -snf /etc/init.d/${initd} "${ROOT}"/etc/runlevels/boot/${initd} +} +add_boot_init_mit_config() { + local config=$1 initd=$2 + if [[ -e ${ROOT}${config} ]] ; then + if [[ -n $(sed -e 's:#.*::' -e '/^[[:space:]]*$/d' "${ROOT}"/${config}) ]] ; then + add_boot_init ${initd} + fi + fi +} + +pkg_preinst() { + local f + + # default net script is just comments, so no point in biting people + # in the ass by accident + mv "${D}"/etc/conf.d/net "${T}"/ + [[ -e ${ROOT}/etc/conf.d/net ]] && cp "${ROOT}"/etc/conf.d/net "${T}"/ + + # upgrade timezone file ... do it before moving clock + if [[ -e ${ROOT}/etc/conf.d/clock && ! -e ${ROOT}/etc/timezone ]] ; then + ( + source "${ROOT}"/etc/conf.d/clock + [[ -n ${TIMEZONE} ]] && echo "${TIMEZONE}" > "${ROOT}"/etc/timezone + ) + fi + + # /etc/conf.d/clock moved to /etc/conf.d/hwclock + local clock + use kernel_FreeBSD && clock="adjkerntz" || clock="hwclock" + if [[ -e ${ROOT}/etc/conf.d/clock ]] ; then + mv "${ROOT}"/etc/conf.d/clock "${ROOT}"/etc/conf.d/${clock} + fi + if [[ -L ${ROOT}/etc/runlevels/boot/clock ]] ; then + rm -f "${ROOT}"/etc/runlevels/boot/clock + ln -snf /etc/init.d/${clock} "${ROOT}"/etc/runlevels/boot/${clock} + fi + if [[ -L ${ROOT}${LIBDIR}/rc/init.d/started/clock ]] ; then + rm -f "${ROOT}${LIBDIR}"/rc/init.d/started/clock + ln -snf /etc/init.d/${clock} "${ROOT}${LIBDIR}"/rc/init.d/started/${clock} + fi + + # /etc/conf.d/rc is no longer used for configuration + if [[ -e ${ROOT}/etc/conf.d/rc ]] ; then + elog "/etc/conf.d/rc is no longer used for configuration." + elog "Please migrate your settings to /etc/rc.conf as applicable" + elog "and delete /etc/conf.d/rc" + fi + + # force net init.d scripts into symlinks + for f in "${ROOT}"/etc/init.d/net.* ; do + [[ -e ${f} ]] || continue # catch net.* not matching anything + [[ ${f} == */net.lo ]] && continue # real file now + [[ ${f} == *.openrc.bak ]] && continue + if [[ ! -L ${f} ]] ; then + elog "Moved net service '${f##*/}' to '${f##*/}.openrc.bak' to force a symlink." + elog "You should delete '${f##*/}.openrc.bak' if you don't need it." + mv "${f}" "${f}.openrc.bak" + ln -snf net.lo "${f}" + fi + done + + # termencoding was added in 0.2.1 and needed in boot + has_version ">=sys-apps/openrc-0.2.1" || add_boot_init termencoding + + # skip remaining migration if we already have openrc installed + has_version sys-apps/openrc && return 0 + + # baselayout boot init scripts have been split out + for f in $(cd "${D}"/usr/share/${PN}/runlevels/boot || exit; echo *) ; do + add_boot_init ${f} + done + + # Try to auto-add some addons when possible + add_boot_init_mit_config /etc/conf.d/cryptfs dmcrypt + add_boot_init_mit_config /etc/conf.d/dmcrypt dmcrypt + add_boot_init_mit_config /etc/mdadm.conf mdraid + add_boot_init_mit_config /etc/evms.conf evms + [[ -e ${ROOT}/sbin/dmsetup ]] && add_boot_init device-mapper + [[ -e ${ROOT}/sbin/vgscan ]] && add_boot_init lvm + elog "Add on services (such as RAID/dmcrypt/LVM/etc...) are now stand alone" + elog "init.d scripts. If you use such a thing, make sure you have the" + elog "required init.d scripts added to your boot runlevel." + + # Upgrade out state for baselayout-1 users + if [[ ! -e ${ROOT}${LIBDIR}/rc/init.d/started ]] ; then + ( + [[ -e ${ROOT}/etc/conf.d/rc ]] && source "${ROOT}"/etc/conf.d/rc + svcdir=${svcdir:-/var/lib/init.d} + if [[ ! -d ${ROOT}${svcdir}/started ]] ; then + ewarn "No state found, and no state exists" + elog "You should reboot this host" + else + mkdir -p "${ROOT}${LIBDIR}/rc/init.d" + einfo "Moving state from ${ROOT}${svcdir} to ${ROOT}${LIBDIR}/rc/init.d" + mv "${ROOT}${svcdir}"/* "${ROOT}${LIBDIR}"/rc/init.d + rm -rf "${ROOT}${LIBDIR}"/rc/init.d/daemons \ + "${ROOT}${LIBDIR}"/rc/init.d/console + umount "${ROOT}${svcdir}" 2>/dev/null + rm -rf "${ROOT}${svcdir}" + fi + ) + fi + + # Handle the /etc/modules.autoload.d -> /etc/conf.d/modules transition + if [[ -d ${ROOT}/etc/modules.autoload.d ]] ; then + elog "Converting your /etc/modules.autoload.d/ files to /etc/conf.d/modules" + rm -f "${ROOT}"/etc/modules.autoload.d/.keep* + rmdir "${ROOT}"/etc/modules.autoload.d 2>/dev/null + if [[ -d ${ROOT}/etc/modules.autoload.d ]] ; then + local f v + for f in "${ROOT}"/etc/modules.autoload.d/* ; do + v=${f##*/} + v=${v#kernel-} + v=${v//[^[:alnum:]]/_} + gawk -v v="${v}" -v f="${f##*/}" ' + BEGIN { print "\n### START: Auto-converted from " f "\n" } + { + if ($0 ~ /^[^#]/) { + print "modules_" v "=\"${modules_" v "} " $1 "\"" + gsub(/[^[:alnum:]]/, "_", $1) + printf "module_" $1 "_args_" v "=\"" + for (i = 2; i <= NF; ++i) { + if (i > 2) + printf " " + printf $i + } + print "\"\n" + } else + print + } + END { print "\n### END: Auto-converted from " f "\n" } + ' "${f}" >> "${D}"/etc/conf.d/modules + done + rm -f "${f}" + rmdir "${ROOT}"/etc/modules.autoload.d 2>/dev/null + fi + fi +} + +pkg_postinst() { + # Remove old baselayout links + rm -f "${ROOT}"/etc/runlevels/boot/{check{fs,root},rmnologin} + + [[ -e ${T}/net && ! -e ${ROOT}/etc/conf.d/net ]] && mv "${T}"/net "${ROOT}"/etc/conf.d/net + + # Make our runlevels if they don't exist + if [[ ! -e ${ROOT}/etc/runlevels ]] ; then + einfo "Copying across default runlevels" + cp -RPp "${ROOT}"/usr/share/${PN}/runlevels "${ROOT}"/etc + fi + + # update the dependency tree bug #224171 + [[ "${ROOT}" = "/" ]] && "${ROOT}/${LIBDIR}"/rc/bin/rc-depend -u + + if [[ -d ${ROOT}/etc/modules.autoload.d ]] ; then + ewarn "/etc/modules.autoload.d is no longer used. Please convert" + ewarn "your files to /etc/conf.d/modules and delete the directory." + fi + + elog "You should now update all files in /etc, using etc-update" + elog "or equivalent before restarting any services or this host." + elog + elog "Please read the migration guide available at:" + elog "http://www.gentoo.org/doc/en/openrc-migration.xml" +} diff --git a/overlay/sys-cluster/beowulf-head/Manifest b/overlay/sys-cluster/beowulf-head/Manifest index 618925d..f043833 100644 --- a/overlay/sys-cluster/beowulf-head/Manifest +++ b/overlay/sys-cluster/beowulf-head/Manifest @@ -1,3 +1,9 @@ +AUX c3-add 686 RMD160 ede4fc8bb832046607e79f831942a3d0708cbcab SHA1 cb496f8137065e2559caea38d932f3a07340d170 SHA256 e0195e7b6dcc77465b768c746825bcfec0157c3a6e5e0e4e1f1336521ecef0d6 +AUX c3-del 104 RMD160 18b49c1a929b0fc08912692f41ec8df1313c940d SHA1 b7d66c89d64fcb7a32a8d62cb147dcaf2994c3e7 SHA256 f4e3e9431c40b50f1596816eaa4d1f760ab840ae555d5fa2a76a7672d0432dfa AUX cluster.conf 2393 RMD160 2ccb24e8f2cb83d8a002fc239d0be6bb7968ed4d SHA1 ad4cc8ff8b8f2f4270755f387228efb0cc63c048 SHA256 660478b6763cc3c6190541bb46d5d17a01a5e366ac8ce84551d1ae31f2eb1673 -EBUILD beowulf-head-0.1.ebuild 6118 RMD160 603098747d9f6364ebbf366db57aff0bc2ab2f02 SHA1 a7e7bfad77e95ab6a872ee34a4412c4e57255fb3 SHA256 7220faed9f98222b04e6aa182d2bb9a6539db5ea59756980c2b7ed3be0960cf4 +AUX dnsmasq-2.4-conf 1552 RMD160 08dae8ff1cd345acedb4cbbb819f8ccf35211b8b SHA1 e770ba62ca66b61cd84409bbb22433b1dfcf9ff3 SHA256 f2b63d6fc8558c84e79b5f172c4057354bdfe000aae19f8aa1857655259a6b7f +AUX node-manager 1574 RMD160 bda5378e456c567947504fc95a3db4163c214517 SHA1 d1ca1142b31019b9edc3827dc5c9c9ad7b1ed996 SHA256 fd2f94e4f58aea91f0d11ea87f6f18f160782721956455f3892dae5e435827db +AUX torque-add 183 RMD160 30eae769ed05dabdc144d1e8259fd14f1f97a152 SHA1 dc2ece4b1a890557193b7dfad28e8696af7529c2 SHA256 d4ea4b509d47044f2e445b0fd36beb64eb35143ac62ccbaa4ac8905b8a12746e +AUX torque-del 132 RMD160 bc1e0a9ccc6ce20630003f7372d62bdb705b9187 SHA1 c735f03fb6715ab601cb84f4fbc92be78ce43575 SHA256 c455764846f192930625721958532d3cee64aa75d9d83a37b67544e4c7bdee64 +EBUILD beowulf-head-0.1.ebuild 7273 RMD160 966e7332fcfdac3560b29e7e8be7dff866630c2a SHA1 911e996fae639424abec535b05e256091f13e61c SHA256 ef70453c432338f42ba9b6ccb7aa4a0a591fef5c9a7935cfd516761d73fe9eae MISC ChangeLog 101 RMD160 d735719ccc09237e68e4e984eb5451000b06c382 SHA1 79098d1e8ae451f0430d363da6b717966ade6e78 SHA256 8dd010e2772026c4a6b86839842931b99c85c44a5e9f380864cc275b55587ac2 diff --git a/overlay/sys-cluster/beowulf-head/beowulf-head-0.1.ebuild b/overlay/sys-cluster/beowulf-head/beowulf-head-0.1.ebuild index bdc1252..4e85bff 100644 --- a/overlay/sys-cluster/beowulf-head/beowulf-head-0.1.ebuild +++ b/overlay/sys-cluster/beowulf-head/beowulf-head-0.1.ebuild @@ -9,7 +9,7 @@ SRC_URI="" LICENSE="GPL-2" SLOT="0" KEYWORDS="amd64 ~x86" -IUSE="autofs nonfsv4" +IUSE="autofs nonfsv4 beep pbs" DEPEND="" RDEPEND="net-nds/ldap-auth @@ -18,34 +18,47 @@ RDEPEND="net-nds/ldap-auth net-misc/sipcalc net-fs/nfs-utils sys-cluster/c3 - net-dns/openresolv" + net-dns/openresolv + beep? (app-misc/beep) + pbs? (sys-cluster/torque)" CONFPATH="/etc/gentoo" CONFIG_FILE="cluster.conf" +MODULESPATH="/usr/libexec/clustering/" +DNSMASQSCRIPT="/usr/sbin/node-manager" inherit confutils get_ip_config() { CLUSTER_ETH_IP="$(sipcalc ${CLUSTER_ETH} | egrep 'Host address\W*-' | cut -d- -f2)" - RANGE_START="$(sipcalc ${CLUSTER_ETH} | egrep 'Host address\W*-' | cut -d- -f2)" - IPSTART="${RANGE_START%.*}.$(( ${RANGE_START##*.} +1 ))" - IPSTOP="$( sipcalc ${CLUSTER_ETH} | egrep 'Usable range\W*-' | cut -d- -f3)" - SUBNET="$( sipcalc ${CLUSTER_ETH} | egrep 'Network mask\W*-' | cut -d- -f2)" - NETADDR="$( sipcalc ${CLUSTER_ETH} | egrep 'Network address' | cut -d- -f2)" + IPSTART="${CLUSTER_ETH_IP%.*}.$(( ${CLUSTER_ETH_IP##*.} +1 ))" + IPSTOP=" $(sipcalc ${CLUSTER_ETH} | egrep 'Usable range\W*-' | cut -d- -f3)" + SUBNET=" $(sipcalc ${CLUSTER_ETH} | egrep 'Network mask\W*-' | cut -d- -f2)" + NETADDR=" $(sipcalc ${CLUSTER_ETH} | egrep 'Network address' | cut -d- -f2)" NODECOUNT="$(sipcalc ${CLUSTER_ETH} | egrep 'Addresses in network'| cut -d- -f2)" # We have to clean up the vars from spaces: CLUSTER_ETH_IP=${CLUSTER_ETH_IP// } - RANGE_START=${RANGE_START// } IPSTART=${IPSTART// } IPSTOP=${IPSTOP// } SUBNET=${SUBNET// } NETADDR=${NETADDR// } + NODECOUNT=$(($NODECOUNT-2)) # removing server host + broadcast } dnsmasq_conf() { +# The following (commneted) would be much cleaner but this can't work in the livecd environment since the portage tree doesn't exist +# and the template file is thus non-existant (${FILESDIR}/...) . +# cp ${FILESDIR}/dnsmasq-2.4-conf ${ROOT}/etc/dnsmasq.conf +# +# for I in PUBETH LOCALDOMAINNAME TFTPROOT IPSTART IPSTOP SUBNET CLUSTER_ETH_IP PXELINUX HOSTNAME NFSROOT NODECOUNT DNSMASQSCRIPT +# do +# sed -i -e"s:%%$I:${!I}:g" ${ROOT}/etc/dnsmasq.conf +# done +# cp + cat > ${ROOT}/etc/dnsmasq.conf <<-EOF # For the sake of non-duplication of documentation, please view the dnsmasq manpage for a description of these options. # Public interface @@ -62,6 +75,8 @@ dnsmasq_conf() # We need the following to retain dhcp served resolution info: resolv-file=/etc/dnsmasq-resolv.conf + dhcp-script=$DNSMASQSCRIPT + # Some netiquette: domain-needed bogus-priv @@ -190,11 +205,30 @@ pkg_setup() { use autofs && die "We're not there yet ;)" } -src_install() -{ - dodir ${CONFPATH} +src_unpack(){ + mkdir -p ${S} + cp ${FILESDIR}/node-manager ${S}/node-manager || die "Som ting Wong" + sed -ie "s:%%MODULESPATH:$MODULESPATH:" ${S}/node-manager + if use beep ; then + sed -ie "s:%%BEEPS:true:" ${S}/node-manager + else + sed -ie "s:%%BEEPS:false:" ${S}/node-manager + fi +} + +src_install(){ + dodir ${CONFPATH} insinto ${CONFPATH} - doins ${FILESDIR}/${CONFIG_FILE} + doins ${FILESDIR}/${CONFIG_FILE} + + dosbin ${S}/node-manager + + # add and delnode are derived from the dnsmasq add/del/old directives + # passed onto the node-manager script...don't change arbritrarily + dodir ${MODULESPATH} + exeinto ${MODULESPATH} + doexe ${FILESDIR}/c3-* + use pbs && doexe ${FILESDIR}/torque-* elog "You need to edit ${CONFPATH}/${CONFIG_FILE} to your likings" elog "Once you are done, set CONFIG_OK=\"yes\" in there and" diff --git a/overlay/sys-cluster/beowulf-head/files/c3-add b/overlay/sys-cluster/beowulf-head/files/c3-add index 71782e9..b05dc1a 100755 --- a/overlay/sys-cluster/beowulf-head/files/c3-add +++ b/overlay/sys-cluster/beowulf-head/files/c3-add @@ -1,15 +1,15 @@ #!/bin/bash -C3CONF=./c3.conf -TMP=/tmp/$0-$$ +C3CONF=/etc/c3.conf +TMP=/tmp/$(basename $0)-$$ NODES=${TMP}_NODES exec > $TMP shopt -s xpg_echo -echo "# This files was automatically created by $PWD/$0 and shouldn't be manually modified unless $0 is disabled." -echo "# This file descriptor is inefficient for the sake of code simplicity" +echo "# This file was automatically created by $PWD/$0 and shouldn't be manually modified unless $0 is disabled." +echo "# This file structure is inefficient for the sake of code simplicity" # define the cluster echo "cluster gentoo {" @@ -20,7 +20,7 @@ echo "\t$HOSTNAME" # Extract the pre-defined nodes from the existing file egrep -v -e"({|}|#|$HOSTNAME)" $C3CONF > $NODES echo "\t$4" >> $NODES -cat $NODES | sort -g +cat $NODES | sort -g | uniq | egrep -v "(^$|^\W$)" # End of config file: echo "}" diff --git a/overlay/sys-cluster/beowulf-head/files/c3-del b/overlay/sys-cluster/beowulf-head/files/c3-del index 33990ee..af7284e 100755 --- a/overlay/sys-cluster/beowulf-head/files/c3-del +++ b/overlay/sys-cluster/beowulf-head/files/c3-del @@ -1,7 +1,7 @@ #!/bin/bash -C3CONF=./c3.conf -TMP=/tmp/$0-$$ +C3CONF=/etc/c3.conf +TMP=/tmp/$(basename $0)-$$ grep -v $4 $C3CONF > $TMP diff --git a/overlay/sys-cluster/beowulf-head/files/dnsmasq-2.4-conf b/overlay/sys-cluster/beowulf-head/files/dnsmasq-2.4-conf index fa3f257..c99d2df 100644 --- a/overlay/sys-cluster/beowulf-head/files/dnsmasq-2.4-conf +++ b/overlay/sys-cluster/beowulf-head/files/dnsmasq-2.4-conf @@ -13,6 +13,8 @@ expand-hosts # We need the following to retain dhcp served resolution info: resolv-file=/etc/dnsmasq-resolv.conf +dhcp-script=/usr/sbin/node-manager + # Some netiquette: domain-needed bogus-priv diff --git a/overlay/sys-cluster/beowulf-head/files/node-manager b/overlay/sys-cluster/beowulf-head/files/node-manager index 4b6c9b0..a0493bb 100755 --- a/overlay/sys-cluster/beowulf-head/files/node-manager +++ b/overlay/sys-cluster/beowulf-head/files/node-manager @@ -3,7 +3,7 @@ # but it can also be used to manage the manual insertion and deletion of nodes. # Make sure dnsmasq doesn't come an play against you though. -%%MODULESPATH +MODULESPATH=%%MODULESPATH BEEPS=%%BEEPS run_modules(){ @@ -24,8 +24,10 @@ node_is_alive(){ # Nodes can either be added or removed. For a thorough description of the mechanism, see # dnsmasq's manpage section concerning dhcp-script old_node(){ - if [[ ! node_is_alive ]]; then - shift + shift + if [[ node_is_alive ]]; then + $0 add $* + else $0 del $* fi } @@ -35,7 +37,14 @@ usage(){ echo "$0 <add|del|old> <MAC address> <IP address> <hostname>" } -(( $# != 3 )) && { usage; exit 1 }; + +if [[ $# != 4 ]]; then +# We just ignore the call without a hostname +# usage; +# + exit 0; +fi + ##### Variables ####### COMMAND=$1 @@ -50,14 +59,16 @@ STOP=2000 STEP=200 BEEP_add="$(for I in `seq $START $STEP $STOP`; do echo -n " -f $I -l $PAUSE -r 1 -n"; done)" BEEP_del="$(for I in `seq $STOP -$STEP $START`; do echo -n " -f $I -l $PAUSE -r 1 -n"; done)" -BEEP_add="beep ${BEEPS_add%%-n}" -BEEP_del="beep ${BEEPS_del%%-n}" +BEEP_add="beep ${BEEP_add%%-n}" +BEEP_del="beep ${BEEP_del%%-n}" ##### Variables END ### if [[ $COMMAND == "old" ]] then - old_node $@ + # we background this since it performs a ping with a 1second timeout + # on a 256 node system, those seconds add up... + old_node $@ & else run_modules $@ fi diff --git a/overlay/sys-cluster/beowulf-head/files/torque-add b/overlay/sys-cluster/beowulf-head/files/torque-add index 73d8957..01f251d 100755 --- a/overlay/sys-cluster/beowulf-head/files/torque-add +++ b/overlay/sys-cluster/beowulf-head/files/torque-add @@ -1,7 +1,7 @@ #!/bin/bash -TMP=/tmp/$0-$$ +TMP=/tmp/$(basename $0)-$$ NODES=$PBS_SERVER_HOME/server_priv/nodes echo $4 >> $TMP -cat $TMP | sort -g > $NODES - +cat $NODES $TMP | sort -g | uniq | egrep -v "(^$|^\W$)" > $NODES. +mv $NODES.$$ $NODES diff --git a/overlay/sys-cluster/beowulf-head/files/torque-del b/overlay/sys-cluster/beowulf-head/files/torque-del index 09a8935..5648a07 100755 --- a/overlay/sys-cluster/beowulf-head/files/torque-del +++ b/overlay/sys-cluster/beowulf-head/files/torque-del @@ -1,7 +1,7 @@ #!/bin/bash -TMP=/tmp/$0-$$ +TMP=/tmp/$(basename $0)-$$ NODES=$PBS_SERVER_HOME/server_priv/nodes -grep -v $4 $NODES > $TMP -mv $TMP $NODES +grep -v $4 $NODES > $NODES.$$ +mv $NODES.$$ $NODES diff --git a/profiles/hpc-clustering/package.keywords/nss_ldap b/profiles/hpc-clustering/package.keywords/nss_ldap new file mode 100644 index 0000000..74b5d6c --- /dev/null +++ b/profiles/hpc-clustering/package.keywords/nss_ldap @@ -0,0 +1,2 @@ +=sys-auth/nss_ldap-259 + diff --git a/profiles/hpc-clustering/package.keywords/strace b/profiles/hpc-clustering/package.keywords/strace new file mode 100644 index 0000000..5b26ca6 --- /dev/null +++ b/profiles/hpc-clustering/package.keywords/strace @@ -0,0 +1,2 @@ +=dev-libs/elfutils-0.131-r2 + diff --git a/profiles/hpc-clustering/package.keywords/torque b/profiles/hpc-clustering/package.keywords/torque new file mode 100644 index 0000000..8c0d81b --- /dev/null +++ b/profiles/hpc-clustering/package.keywords/torque @@ -0,0 +1 @@ +=sys-cluster/torque-2.3.3 ** diff --git a/profiles/hpc-clustering/package.use b/profiles/hpc-clustering/package.use index 3d39a38..eb44b25 100644 --- a/profiles/hpc-clustering/package.use +++ b/profiles/hpc-clustering/package.use @@ -15,4 +15,5 @@ app-admin/sudo ldap # ^^^ Nope...that won't work, so commenting and trying in livecd-stage1.spec instead (net-nds/openldap added to the list) sys-cluster/openmpi smp threads fortran -app-editors/vim -minimal +app-editors/vim-core -minimal -livecd +app-editors/vim -minimal -livecd |