1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
|
# portage: Lock management code
# Copyright 2004-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \
"hardlock_name", "hardlink_is_mine", "hardlink_lockfile", \
"unhardlink_lockfile", "hardlock_cleanup"]
import errno
import fcntl
import multiprocessing
import sys
import tempfile
import time
import warnings
import portage
from portage import os, _encodings, _unicode_decode
from portage.exception import (DirectoryNotFound, FileNotFound,
InvalidData, TryAgain, OperationNotPermitted, PermissionDenied,
ReadOnlyFileSystem)
from portage.util import writemsg
from portage.util.install_mask import _raise_exc
from portage.localization import _
if sys.hexversion >= 0x3000000:
# pylint: disable=W0622
basestring = str
HARDLINK_FD = -2
_HARDLINK_POLL_LATENCY = 3 # seconds
# Used by emerge in order to disable the "waiting for lock" message
# so that it doesn't interfere with the status display.
_quiet = False
_lock_fn = None
def _get_lock_fn():
"""
Returns fcntl.lockf if proven to work, and otherwise returns fcntl.flock.
On some platforms fcntl.lockf is known to be broken.
"""
global _lock_fn
if _lock_fn is not None:
return _lock_fn
def _test_lock(fd, lock_path):
os.close(fd)
try:
with open(lock_path, 'a') as f:
fcntl.lockf(f.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
except EnvironmentError as e:
if e.errno == errno.EAGAIN:
# Parent process holds lock, as expected.
sys.exit(0)
# Something went wrong.
sys.exit(1)
fd, lock_path = tempfile.mkstemp()
try:
try:
fcntl.lockf(fd, fcntl.LOCK_EX)
except EnvironmentError:
pass
else:
proc = multiprocessing.Process(target=_test_lock,
args=(fd, lock_path))
proc.start()
proc.join()
if proc.exitcode == os.EX_OK:
# Use fcntl.lockf because the test passed.
_lock_fn = fcntl.lockf
return _lock_fn
finally:
os.close(fd)
os.unlink(lock_path)
# Fall back to fcntl.flock.
_lock_fn = fcntl.flock
return _lock_fn
_open_fds = set()
def _close_fds():
"""
This is intended to be called after a fork, in order to close file
descriptors for locks held by the parent process. This can be called
safely after a fork without exec, unlike the _setup_pipes close_fds
behavior.
"""
while _open_fds:
os.close(_open_fds.pop())
def lockdir(mydir, flags=0):
return lockfile(mydir, wantnewlockfile=1, flags=flags)
def unlockdir(mylock):
return unlockfile(mylock)
def lockfile(mypath, wantnewlockfile=0, unlinkfile=0,
waiting_msg=None, flags=0):
"""
If wantnewlockfile is True then this creates a lockfile in the parent
directory as the file: '.' + basename + '.portage_lockfile'.
"""
if not mypath:
raise InvalidData(_("Empty path given"))
# Since Python 3.4, chown requires int type (no proxies).
portage_gid = int(portage.data.portage_gid)
# Support for file object or integer file descriptor parameters is
# deprecated due to ambiguity in whether or not it's safe to close
# the file descriptor, making it prone to "Bad file descriptor" errors
# or file descriptor leaks.
if isinstance(mypath, basestring) and mypath[-1] == '/':
mypath = mypath[:-1]
lockfilename_path = mypath
if hasattr(mypath, 'fileno'):
warnings.warn("portage.locks.lockfile() support for "
"file object parameters is deprecated. Use a file path instead.",
DeprecationWarning, stacklevel=2)
lockfilename_path = getattr(mypath, 'name', None)
mypath = mypath.fileno()
if isinstance(mypath, int):
warnings.warn("portage.locks.lockfile() support for integer file "
"descriptor parameters is deprecated. Use a file path instead.",
DeprecationWarning, stacklevel=2)
lockfilename = mypath
wantnewlockfile = 0
unlinkfile = 0
elif wantnewlockfile:
base, tail = os.path.split(mypath)
lockfilename = os.path.join(base, "." + tail + ".portage_lockfile")
lockfilename_path = lockfilename
unlinkfile = 1
else:
lockfilename = mypath
if isinstance(mypath, basestring):
if not os.path.exists(os.path.dirname(mypath)):
raise DirectoryNotFound(os.path.dirname(mypath))
preexisting = os.path.exists(lockfilename)
old_mask = os.umask(000)
try:
while True:
try:
myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
except OSError as e:
if e.errno in (errno.ENOENT, errno.ESTALE) and os.path.isdir(os.path.dirname(lockfilename)):
# Retry required for NFS (see bug 636798).
continue
else:
_raise_exc(e)
else:
break
if not preexisting:
try:
if portage.data.secpass >= 1 and os.stat(lockfilename).st_gid != portage_gid:
os.chown(lockfilename, -1, portage_gid)
except OSError as e:
if e.errno in (errno.ENOENT, errno.ESTALE):
return lockfile(mypath,
wantnewlockfile=wantnewlockfile,
unlinkfile=unlinkfile, waiting_msg=waiting_msg,
flags=flags)
else:
writemsg("%s: chown('%s', -1, %d)\n" % \
(e, lockfilename, portage_gid), noiselevel=-1)
writemsg(_("Cannot chown a lockfile: '%s'\n") % \
lockfilename, noiselevel=-1)
writemsg(_("Group IDs of current user: %s\n") % \
" ".join(str(n) for n in os.getgroups()),
noiselevel=-1)
finally:
os.umask(old_mask)
elif isinstance(mypath, int):
myfd = mypath
else:
raise ValueError(_("Unknown type passed in '%s': '%s'") % \
(type(mypath), mypath))
# try for a non-blocking lock, if it's held, throw a message
# we're waiting on lockfile and use a blocking attempt.
locking_method = portage._eintr_func_wrapper(_get_lock_fn())
try:
if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
raise IOError(errno.ENOSYS, "Function not implemented")
locking_method(myfd, fcntl.LOCK_EX|fcntl.LOCK_NB)
except IOError as e:
if not hasattr(e, "errno"):
raise
if e.errno in (errno.EACCES, errno.EAGAIN, errno.ENOLCK):
# resource temp unavailable; eg, someone beat us to the lock.
if flags & os.O_NONBLOCK:
os.close(myfd)
raise TryAgain(mypath)
global _quiet
if _quiet:
out = None
else:
out = portage.output.EOutput()
if waiting_msg is None:
if isinstance(mypath, int):
waiting_msg = _("waiting for lock on fd %i") % myfd
else:
waiting_msg = _("waiting for lock on %s") % lockfilename
if out is not None:
out.ebegin(waiting_msg)
# try for the exclusive lock now.
enolock_msg_shown = False
while True:
try:
locking_method(myfd, fcntl.LOCK_EX)
except EnvironmentError as e:
if e.errno == errno.ENOLCK:
# This is known to occur on Solaris NFS (see
# bug #462694). Assume that the error is due
# to temporary exhaustion of record locks,
# and loop until one becomes available.
if not enolock_msg_shown:
enolock_msg_shown = True
if isinstance(mypath, int):
context_desc = _("Error while waiting "
"to lock fd %i") % myfd
else:
context_desc = _("Error while waiting "
"to lock '%s'") % lockfilename
writemsg("\n!!! %s: %s\n" % (context_desc, e),
noiselevel=-1)
time.sleep(_HARDLINK_POLL_LATENCY)
continue
if out is not None:
out.eend(1, str(e))
raise
else:
break
if out is not None:
out.eend(os.EX_OK)
elif e.errno in (errno.ENOSYS,):
# We're not allowed to lock on this FS.
if not isinstance(lockfilename, int):
# If a file object was passed in, it's not safe
# to close the file descriptor because it may
# still be in use.
os.close(myfd)
lockfilename_path = _unicode_decode(lockfilename_path,
encoding=_encodings['fs'], errors='strict')
if not isinstance(lockfilename_path, basestring):
raise
link_success = hardlink_lockfile(lockfilename_path,
waiting_msg=waiting_msg, flags=flags)
if not link_success:
raise
lockfilename = lockfilename_path
locking_method = None
myfd = HARDLINK_FD
else:
raise
if isinstance(lockfilename, basestring) and \
myfd != HARDLINK_FD and _lockfile_was_removed(myfd, lockfilename):
# The file was deleted on us... Keep trying to make one...
os.close(myfd)
writemsg(_("lockfile recurse\n"), 1)
lockfilename, myfd, unlinkfile, locking_method = lockfile(
mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile,
waiting_msg=waiting_msg, flags=flags)
if myfd != HARDLINK_FD:
# FD_CLOEXEC is enabled by default in Python >=3.4.
if sys.hexversion < 0x3040000:
try:
fcntl.FD_CLOEXEC
except AttributeError:
pass
else:
fcntl.fcntl(myfd, fcntl.F_SETFD,
fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
_open_fds.add(myfd)
writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1)
return (lockfilename, myfd, unlinkfile, locking_method)
def _lockfile_was_removed(lock_fd, lock_path):
"""
Check if lock_fd still refers to a file located at lock_path, since
the file may have been removed by a concurrent process that held the
lock earlier. This implementation includes support for NFS, where
stat is not reliable for removed files due to the default file
attribute cache behavior ('ac' mount option).
@param lock_fd: an open file descriptor for a lock file
@type lock_fd: int
@param lock_path: path of lock file
@type lock_path: str
@rtype: bool
@return: True if lock_path exists and corresponds to lock_fd, False otherwise
"""
try:
fstat_st = os.fstat(lock_fd)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
# Since stat is not reliable for removed files on NFS with the default
# file attribute cache behavior ('ac' mount option), create a temporary
# hardlink in order to prove that the file path exists on the NFS server.
hardlink_path = hardlock_name(lock_path)
try:
os.unlink(hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
try:
try:
os.link(lock_path, hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return True
hardlink_stat = os.stat(hardlink_path)
if hardlink_stat.st_ino != fstat_st.st_ino or hardlink_stat.st_dev != fstat_st.st_dev:
return True
finally:
try:
os.unlink(hardlink_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
_raise_exc(e)
return False
def _fstat_nlink(fd):
"""
@param fd: an open file descriptor
@type fd: Integer
@rtype: Integer
@return: the current number of hardlinks to the file
"""
try:
return os.fstat(fd).st_nlink
except EnvironmentError as e:
if e.errno in (errno.ENOENT, errno.ESTALE):
# Some filesystems such as CIFS return
# ENOENT which means st_nlink == 0.
return 0
raise
def unlockfile(mytuple):
#XXX: Compatability hack.
if len(mytuple) == 3:
lockfilename, myfd, unlinkfile = mytuple
locking_method = fcntl.flock
elif len(mytuple) == 4:
lockfilename, myfd, unlinkfile, locking_method = mytuple
else:
raise InvalidData
if(myfd == HARDLINK_FD):
unhardlink_lockfile(lockfilename, unlinkfile=unlinkfile)
return True
# myfd may be None here due to myfd = mypath in lockfile()
if isinstance(lockfilename, basestring) and \
not os.path.exists(lockfilename):
writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1)
if myfd is not None:
os.close(myfd)
_open_fds.remove(myfd)
return False
try:
if myfd is None:
myfd = os.open(lockfilename, os.O_WRONLY, 0o660)
unlinkfile = 1
locking_method(myfd, fcntl.LOCK_UN)
except OSError:
if isinstance(lockfilename, basestring):
os.close(myfd)
_open_fds.remove(myfd)
raise IOError(_("Failed to unlock file '%s'\n") % lockfilename)
try:
# This sleep call was added to allow other processes that are
# waiting for a lock to be able to grab it before it is deleted.
# lockfile() already accounts for this situation, however, and
# the sleep here adds more time than is saved overall, so am
# commenting until it is proved necessary.
#time.sleep(0.0001)
if unlinkfile:
locking_method(myfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
# We won the lock, so there isn't competition for it.
# We can safely delete the file.
writemsg(_("Got the lockfile...\n"), 1)
if _fstat_nlink(myfd) == 1:
os.unlink(lockfilename)
writemsg(_("Unlinked lockfile...\n"), 1)
locking_method(myfd, fcntl.LOCK_UN)
else:
writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1)
os.close(myfd)
_open_fds.remove(myfd)
return False
except SystemExit:
raise
except Exception as e:
writemsg(_("Failed to get lock... someone took it.\n"), 1)
writemsg(str(e) + "\n", 1)
# why test lockfilename? because we may have been handed an
# fd originally, and the caller might not like having their
# open fd closed automatically on them.
if isinstance(lockfilename, basestring):
os.close(myfd)
_open_fds.remove(myfd)
return True
def hardlock_name(path):
base, tail = os.path.split(path)
return os.path.join(base, ".%s.hardlock-%s-%s" %
(tail, os.uname()[1], os.getpid()))
def hardlink_is_mine(link, lock):
try:
lock_st = os.stat(lock)
if lock_st.st_nlink == 2:
link_st = os.stat(link)
return lock_st.st_ino == link_st.st_ino and \
lock_st.st_dev == link_st.st_dev
except OSError:
pass
return False
def hardlink_lockfile(lockfilename, max_wait=DeprecationWarning,
waiting_msg=None, flags=0):
"""Does the NFS, hardlink shuffle to ensure locking on the disk.
We create a PRIVATE hardlink to the real lockfile, that is just a
placeholder on the disk.
If our file can 2 references, then we have the lock. :)
Otherwise we lather, rise, and repeat.
"""
if max_wait is not DeprecationWarning:
warnings.warn("The 'max_wait' parameter of "
"portage.locks.hardlink_lockfile() is now unused. Use "
"flags=os.O_NONBLOCK instead.",
DeprecationWarning, stacklevel=2)
global _quiet
out = None
displayed_waiting_msg = False
preexisting = os.path.exists(lockfilename)
myhardlock = hardlock_name(lockfilename)
# Since Python 3.4, chown requires int type (no proxies).
portage_gid = int(portage.data.portage_gid)
# myhardlock must not exist prior to our link() call, and we can
# safely unlink it since its file name is unique to our PID
try:
os.unlink(myhardlock)
except OSError as e:
if e.errno in (errno.ENOENT, errno.ESTALE):
pass
else:
func_call = "unlink('%s')" % myhardlock
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
else:
raise
while True:
# create lockfilename if it doesn't exist yet
try:
myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660)
except OSError as e:
func_call = "open('%s')" % lockfilename
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
elif e.errno == ReadOnlyFileSystem.errno:
raise ReadOnlyFileSystem(func_call)
else:
raise
else:
myfd_st = None
try:
myfd_st = os.fstat(myfd)
if not preexisting:
# Don't chown the file if it is preexisting, since we
# want to preserve existing permissions in that case.
if portage.data.secpass >= 1 and myfd_st.st_gid != portage_gid:
os.fchown(myfd, -1, portage_gid)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
writemsg("%s: fchown('%s', -1, %d)\n" % \
(e, lockfilename, portage_gid), noiselevel=-1)
writemsg(_("Cannot chown a lockfile: '%s'\n") % \
lockfilename, noiselevel=-1)
writemsg(_("Group IDs of current user: %s\n") % \
" ".join(str(n) for n in os.getgroups()),
noiselevel=-1)
else:
# another process has removed the file, so we'll have
# to create it again
continue
finally:
os.close(myfd)
# If fstat shows more than one hardlink, then it's extremely
# unlikely that the following link call will result in a lock,
# so optimize away the wasteful link call and sleep or raise
# TryAgain.
if myfd_st is not None and myfd_st.st_nlink < 2:
try:
os.link(lockfilename, myhardlock)
except OSError as e:
func_call = "link('%s', '%s')" % (lockfilename, myhardlock)
if e.errno == OperationNotPermitted.errno:
raise OperationNotPermitted(func_call)
elif e.errno == PermissionDenied.errno:
raise PermissionDenied(func_call)
elif e.errno in (errno.ESTALE, errno.ENOENT):
# another process has removed the file, so we'll have
# to create it again
continue
else:
raise
else:
if hardlink_is_mine(myhardlock, lockfilename):
if out is not None:
out.eend(os.EX_OK)
break
try:
os.unlink(myhardlock)
except OSError as e:
# This should not happen, since the file name of
# myhardlock is unique to our host and PID,
# and the above link() call succeeded.
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
raise FileNotFound(myhardlock)
if flags & os.O_NONBLOCK:
raise TryAgain(lockfilename)
if out is None and not _quiet:
out = portage.output.EOutput()
if out is not None and not displayed_waiting_msg:
displayed_waiting_msg = True
if waiting_msg is None:
waiting_msg = _("waiting for lock on %s\n") % lockfilename
out.ebegin(waiting_msg)
time.sleep(_HARDLINK_POLL_LATENCY)
return True
def unhardlink_lockfile(lockfilename, unlinkfile=True):
myhardlock = hardlock_name(lockfilename)
if unlinkfile and hardlink_is_mine(myhardlock, lockfilename):
# Make sure not to touch lockfilename unless we really have a lock.
try:
os.unlink(lockfilename)
except OSError:
pass
try:
os.unlink(myhardlock)
except OSError:
pass
def hardlock_cleanup(path, remove_all_locks=False):
myhost = os.uname()[1]
mydl = os.listdir(path)
results = []
mycount = 0
mylist = {}
for x in mydl:
if os.path.isfile(path + "/" + x):
parts = x.split(".hardlock-")
if len(parts) == 2:
filename = parts[0][1:]
hostpid = parts[1].split("-")
host = "-".join(hostpid[:-1])
pid = hostpid[-1]
if filename not in mylist:
mylist[filename] = {}
if host not in mylist[filename]:
mylist[filename][host] = []
mylist[filename][host].append(pid)
mycount += 1
results.append(_("Found %(count)s locks") % {"count": mycount})
for x in mylist:
if myhost in mylist[x] or remove_all_locks:
mylockname = hardlock_name(path + "/" + x)
if hardlink_is_mine(mylockname, path + "/" + x) or \
not os.path.exists(path + "/" + x) or \
remove_all_locks:
for y in mylist[x]:
for z in mylist[x][y]:
filename = path + "/." + x + ".hardlock-" + y + "-" + z
if filename == mylockname:
continue
try:
# We're sweeping through, unlinking everyone's locks.
os.unlink(filename)
results.append(_("Unlinked: ") + filename)
except OSError:
pass
try:
os.unlink(path + "/" + x)
results.append(_("Unlinked: ") + path + "/" + x)
os.unlink(mylockname)
results.append(_("Unlinked: ") + mylockname)
except OSError:
pass
else:
try:
os.unlink(mylockname)
results.append(_("Unlinked: ") + mylockname)
except OSError:
pass
return results
|