tests/docker: add python3-setuptools the docker images
[qemu.git] / tests / docker / docker.py
1 #!/usr/bin/env python3
2 #
3 # Docker controlling module
4 #
5 # Copyright (c) 2016 Red Hat Inc.
6 #
7 # Authors:
8 # Fam Zheng <famz@redhat.com>
9 #
10 # This work is licensed under the terms of the GNU GPL, version 2
11 # or (at your option) any later version. See the COPYING file in
12 # the top-level directory.
13
14 import os
15 import sys
16 import subprocess
17 import json
18 import hashlib
19 import atexit
20 import uuid
21 import argparse
22 import enum
23 import tempfile
24 import re
25 import signal
26 from tarfile import TarFile, TarInfo
27 from io import StringIO, BytesIO
28 from shutil import copy, rmtree
29 from pwd import getpwuid
30 from datetime import datetime, timedelta
31
32
33 FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy']
34
35
36 DEVNULL = open(os.devnull, 'wb')
37
38 class EngineEnum(enum.IntEnum):
39 AUTO = 1
40 DOCKER = 2
41 PODMAN = 3
42
43 def __str__(self):
44 return self.name.lower()
45
46 def __repr__(self):
47 return str(self)
48
49 @staticmethod
50 def argparse(s):
51 try:
52 return EngineEnum[s.upper()]
53 except KeyError:
54 return s
55
56
57 USE_ENGINE = EngineEnum.AUTO
58
59 def _bytes_checksum(bytes):
60 """Calculate a digest string unique to the text content"""
61 return hashlib.sha1(bytes).hexdigest()
62
63 def _text_checksum(text):
64 """Calculate a digest string unique to the text content"""
65 return _bytes_checksum(text.encode('utf-8'))
66
67 def _read_dockerfile(path):
68 return open(path, 'rt', encoding='utf-8').read()
69
70 def _file_checksum(filename):
71 return _bytes_checksum(open(filename, 'rb').read())
72
73
74 def _guess_engine_command():
75 """ Guess a working engine command or raise exception if not found"""
76 commands = []
77
78 if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.PODMAN]:
79 commands += [["podman"]]
80 if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.DOCKER]:
81 commands += [["docker"], ["sudo", "-n", "docker"]]
82 for cmd in commands:
83 try:
84 # docker version will return the client details in stdout
85 # but still report a status of 1 if it can't contact the daemon
86 if subprocess.call(cmd + ["version"],
87 stdout=DEVNULL, stderr=DEVNULL) == 0:
88 return cmd
89 except OSError:
90 pass
91 commands_txt = "\n".join([" " + " ".join(x) for x in commands])
92 raise Exception("Cannot find working engine command. Tried:\n%s" %
93 commands_txt)
94
95
96 def _copy_with_mkdir(src, root_dir, sub_path='.'):
97 """Copy src into root_dir, creating sub_path as needed."""
98 dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
99 try:
100 os.makedirs(dest_dir)
101 except OSError:
102 # we can safely ignore already created directories
103 pass
104
105 dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
106 copy(src, dest_file)
107
108
109 def _get_so_libs(executable):
110 """Return a list of libraries associated with an executable.
111
112 The paths may be symbolic links which would need to be resolved to
113 ensure the right data is copied."""
114
115 libs = []
116 ldd_re = re.compile(r"(?:\S+ => )?(\S*) \(:?0x[0-9a-f]+\)")
117 try:
118 ldd_output = subprocess.check_output(["ldd", executable]).decode('utf-8')
119 for line in ldd_output.split("\n"):
120 search = ldd_re.search(line)
121 if search:
122 try:
123 libs.append(s.group(1))
124 except IndexError:
125 pass
126 except subprocess.CalledProcessError:
127 print("%s had no associated libraries (static build?)" % (executable))
128
129 return libs
130
131
132 def _copy_binary_with_libs(src, bin_dest, dest_dir):
133 """Maybe copy a binary and all its dependent libraries.
134
135 If bin_dest isn't set we only copy the support libraries because
136 we don't need qemu in the docker path to run (due to persistent
137 mapping). Indeed users may get confused if we aren't running what
138 is in the image.
139
140 This does rely on the host file-system being fairly multi-arch
141 aware so the file don't clash with the guests layout.
142 """
143
144 if bin_dest:
145 _copy_with_mkdir(src, dest_dir, os.path.dirname(bin_dest))
146 else:
147 print("only copying support libraries for %s" % (src))
148
149 libs = _get_so_libs(src)
150 if libs:
151 for l in libs:
152 so_path = os.path.dirname(l)
153 real_l = os.path.realpath(l)
154 _copy_with_mkdir(real_l, dest_dir, so_path)
155
156
157 def _check_binfmt_misc(executable):
158 """Check binfmt_misc has entry for executable in the right place.
159
160 The details of setting up binfmt_misc are outside the scope of
161 this script but we should at least fail early with a useful
162 message if it won't work.
163
164 Returns the configured binfmt path and a valid flag. For
165 persistent configurations we will still want to copy and dependent
166 libraries.
167 """
168
169 binary = os.path.basename(executable)
170 binfmt_entry = "/proc/sys/fs/binfmt_misc/%s" % (binary)
171
172 if not os.path.exists(binfmt_entry):
173 print ("No binfmt_misc entry for %s" % (binary))
174 return None, False
175
176 with open(binfmt_entry) as x: entry = x.read()
177
178 if re.search("flags:.*F.*\n", entry):
179 print("binfmt_misc for %s uses persistent(F) mapping to host binary" %
180 (binary))
181 return None, True
182
183 m = re.search("interpreter (\S+)\n", entry)
184 interp = m.group(1)
185 if interp and interp != executable:
186 print("binfmt_misc for %s does not point to %s, using %s" %
187 (binary, executable, interp))
188
189 return interp, True
190
191
192 def _read_qemu_dockerfile(img_name):
193 # special case for Debian linux-user images
194 if img_name.startswith("debian") and img_name.endswith("user"):
195 img_name = "debian-bootstrap"
196
197 df = os.path.join(os.path.dirname(__file__), "dockerfiles",
198 img_name + ".docker")
199 return _read_dockerfile(df)
200
201
202 def _dockerfile_preprocess(df):
203 out = ""
204 for l in df.splitlines():
205 if len(l.strip()) == 0 or l.startswith("#"):
206 continue
207 from_pref = "FROM qemu/"
208 if l.startswith(from_pref):
209 # TODO: Alternatively we could replace this line with "FROM $ID"
210 # where $ID is the image's hex id obtained with
211 # $ docker images $IMAGE --format="{{.Id}}"
212 # but unfortunately that's not supported by RHEL 7.
213 inlining = _read_qemu_dockerfile(l[len(from_pref):])
214 out += _dockerfile_preprocess(inlining)
215 continue
216 out += l + "\n"
217 return out
218
219
220 class Docker(object):
221 """ Running Docker commands """
222 def __init__(self):
223 self._command = _guess_engine_command()
224
225 if "docker" in self._command and "TRAVIS" not in os.environ:
226 os.environ["DOCKER_BUILDKIT"] = "1"
227 self._buildkit = True
228 else:
229 self._buildkit = False
230
231 self._instance = None
232 atexit.register(self._kill_instances)
233 signal.signal(signal.SIGTERM, self._kill_instances)
234 signal.signal(signal.SIGHUP, self._kill_instances)
235
236 def _do(self, cmd, quiet=True, **kwargs):
237 if quiet:
238 kwargs["stdout"] = DEVNULL
239 return subprocess.call(self._command + cmd, **kwargs)
240
241 def _do_check(self, cmd, quiet=True, **kwargs):
242 if quiet:
243 kwargs["stdout"] = DEVNULL
244 return subprocess.check_call(self._command + cmd, **kwargs)
245
246 def _do_kill_instances(self, only_known, only_active=True):
247 cmd = ["ps", "-q"]
248 if not only_active:
249 cmd.append("-a")
250
251 filter = "--filter=label=com.qemu.instance.uuid"
252 if only_known:
253 if self._instance:
254 filter += "=%s" % (self._instance)
255 else:
256 # no point trying to kill, we finished
257 return
258
259 print("filter=%s" % (filter))
260 cmd.append(filter)
261 for i in self._output(cmd).split():
262 self._do(["rm", "-f", i])
263
264 def clean(self):
265 self._do_kill_instances(False, False)
266 return 0
267
268 def _kill_instances(self, *args, **kwargs):
269 return self._do_kill_instances(True)
270
271 def _output(self, cmd, **kwargs):
272 try:
273 return subprocess.check_output(self._command + cmd,
274 stderr=subprocess.STDOUT,
275 encoding='utf-8',
276 **kwargs)
277 except TypeError:
278 # 'encoding' argument was added in 3.6+
279 return subprocess.check_output(self._command + cmd,
280 stderr=subprocess.STDOUT,
281 **kwargs).decode('utf-8')
282
283
284 def inspect_tag(self, tag):
285 try:
286 return self._output(["inspect", tag])
287 except subprocess.CalledProcessError:
288 return None
289
290 def get_image_creation_time(self, info):
291 return json.loads(info)[0]["Created"]
292
293 def get_image_dockerfile_checksum(self, tag):
294 resp = self.inspect_tag(tag)
295 labels = json.loads(resp)[0]["Config"].get("Labels", {})
296 return labels.get("com.qemu.dockerfile-checksum", "")
297
298 def build_image(self, tag, docker_dir, dockerfile,
299 quiet=True, user=False, argv=None, registry=None,
300 extra_files_cksum=[]):
301 if argv is None:
302 argv = []
303
304 # pre-calculate the docker checksum before any
305 # substitutions we make for caching
306 checksum = _text_checksum(_dockerfile_preprocess(dockerfile))
307
308 if registry is not None:
309 sources = re.findall("FROM qemu\/(.*)", dockerfile)
310 # Fetch any cache layers we can, may fail
311 for s in sources:
312 pull_args = ["pull", "%s/qemu/%s" % (registry, s)]
313 if self._do(pull_args, quiet=quiet) != 0:
314 registry = None
315 break
316 # Make substitutions
317 if registry is not None:
318 dockerfile = dockerfile.replace("FROM qemu/",
319 "FROM %s/qemu/" %
320 (registry))
321
322 tmp_df = tempfile.NamedTemporaryFile(mode="w+t",
323 encoding='utf-8',
324 dir=docker_dir, suffix=".docker")
325 tmp_df.write(dockerfile)
326
327 if user:
328 uid = os.getuid()
329 uname = getpwuid(uid).pw_name
330 tmp_df.write("\n")
331 tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
332 (uname, uid, uname))
333
334 tmp_df.write("\n")
335 tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" % (checksum))
336 for f, c in extra_files_cksum:
337 tmp_df.write("LABEL com.qemu.%s-checksum=%s" % (f, c))
338
339 tmp_df.flush()
340
341 build_args = ["build", "-t", tag, "-f", tmp_df.name]
342 if self._buildkit:
343 build_args += ["--build-arg", "BUILDKIT_INLINE_CACHE=1"]
344
345 if registry is not None:
346 pull_args = ["pull", "%s/%s" % (registry, tag)]
347 self._do(pull_args, quiet=quiet)
348 cache = "%s/%s" % (registry, tag)
349 build_args += ["--cache-from", cache]
350 build_args += argv
351 build_args += [docker_dir]
352
353 self._do_check(build_args,
354 quiet=quiet)
355
356 def update_image(self, tag, tarball, quiet=True):
357 "Update a tagged image using "
358
359 self._do_check(["build", "-t", tag, "-"], quiet=quiet, stdin=tarball)
360
361 def image_matches_dockerfile(self, tag, dockerfile):
362 try:
363 checksum = self.get_image_dockerfile_checksum(tag)
364 except Exception:
365 return False
366 return checksum == _text_checksum(_dockerfile_preprocess(dockerfile))
367
368 def run(self, cmd, keep, quiet, as_user=False):
369 label = uuid.uuid4().hex
370 if not keep:
371 self._instance = label
372
373 if as_user:
374 uid = os.getuid()
375 cmd = [ "-u", str(uid) ] + cmd
376 # podman requires a bit more fiddling
377 if self._command[0] == "podman":
378 cmd.insert(0, '--userns=keep-id')
379
380 ret = self._do_check(["run", "--label",
381 "com.qemu.instance.uuid=" + label] + cmd,
382 quiet=quiet)
383 if not keep:
384 self._instance = None
385 return ret
386
387 def command(self, cmd, argv, quiet):
388 return self._do([cmd] + argv, quiet=quiet)
389
390
391 class SubCommand(object):
392 """A SubCommand template base class"""
393 name = None # Subcommand name
394
395 def shared_args(self, parser):
396 parser.add_argument("--quiet", action="store_true",
397 help="Run quietly unless an error occurred")
398
399 def args(self, parser):
400 """Setup argument parser"""
401 pass
402
403 def run(self, args, argv):
404 """Run command.
405 args: parsed argument by argument parser.
406 argv: remaining arguments from sys.argv.
407 """
408 pass
409
410
411 class RunCommand(SubCommand):
412 """Invoke docker run and take care of cleaning up"""
413 name = "run"
414
415 def args(self, parser):
416 parser.add_argument("--keep", action="store_true",
417 help="Don't remove image when command completes")
418 parser.add_argument("--run-as-current-user", action="store_true",
419 help="Run container using the current user's uid")
420
421 def run(self, args, argv):
422 return Docker().run(argv, args.keep, quiet=args.quiet,
423 as_user=args.run_as_current_user)
424
425
426 class BuildCommand(SubCommand):
427 """ Build docker image out of a dockerfile. Arg: <tag> <dockerfile>"""
428 name = "build"
429
430 def args(self, parser):
431 parser.add_argument("--include-executable", "-e",
432 help="""Specify a binary that will be copied to the
433 container together with all its dependent
434 libraries""")
435 parser.add_argument("--extra-files", nargs='*',
436 help="""Specify files that will be copied in the
437 Docker image, fulfilling the ADD directive from the
438 Dockerfile""")
439 parser.add_argument("--add-current-user", "-u", dest="user",
440 action="store_true",
441 help="Add the current user to image's passwd")
442 parser.add_argument("--registry", "-r",
443 help="cache from docker registry")
444 parser.add_argument("-t", dest="tag",
445 help="Image Tag")
446 parser.add_argument("-f", dest="dockerfile",
447 help="Dockerfile name")
448
449 def run(self, args, argv):
450 dockerfile = _read_dockerfile(args.dockerfile)
451 tag = args.tag
452
453 dkr = Docker()
454 if "--no-cache" not in argv and \
455 dkr.image_matches_dockerfile(tag, dockerfile):
456 if not args.quiet:
457 print("Image is up to date.")
458 else:
459 # Create a docker context directory for the build
460 docker_dir = tempfile.mkdtemp(prefix="docker_build")
461
462 # Validate binfmt_misc will work
463 if args.include_executable:
464 qpath, enabled = _check_binfmt_misc(args.include_executable)
465 if not enabled:
466 return 1
467
468 # Is there a .pre file to run in the build context?
469 docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
470 if os.path.exists(docker_pre):
471 stdout = DEVNULL if args.quiet else None
472 rc = subprocess.call(os.path.realpath(docker_pre),
473 cwd=docker_dir, stdout=stdout)
474 if rc == 3:
475 print("Skip")
476 return 0
477 elif rc != 0:
478 print("%s exited with code %d" % (docker_pre, rc))
479 return 1
480
481 # Copy any extra files into the Docker context. These can be
482 # included by the use of the ADD directive in the Dockerfile.
483 cksum = []
484 if args.include_executable:
485 # FIXME: there is no checksum of this executable and the linked
486 # libraries, once the image built any change of this executable
487 # or any library won't trigger another build.
488 _copy_binary_with_libs(args.include_executable,
489 qpath, docker_dir)
490
491 for filename in args.extra_files or []:
492 _copy_with_mkdir(filename, docker_dir)
493 cksum += [(filename, _file_checksum(filename))]
494
495 argv += ["--build-arg=" + k.lower() + "=" + v
496 for k, v in os.environ.items()
497 if k.lower() in FILTERED_ENV_NAMES]
498 dkr.build_image(tag, docker_dir, dockerfile,
499 quiet=args.quiet, user=args.user,
500 argv=argv, registry=args.registry,
501 extra_files_cksum=cksum)
502
503 rmtree(docker_dir)
504
505 return 0
506
507
508 class UpdateCommand(SubCommand):
509 """ Update a docker image with new executables. Args: <tag> <executable>"""
510 name = "update"
511
512 def args(self, parser):
513 parser.add_argument("tag",
514 help="Image Tag")
515 parser.add_argument("executable",
516 help="Executable to copy")
517
518 def run(self, args, argv):
519 # Create a temporary tarball with our whole build context and
520 # dockerfile for the update
521 tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
522 tmp_tar = TarFile(fileobj=tmp, mode='w')
523
524 # Add the executable to the tarball, using the current
525 # configured binfmt_misc path. If we don't get a path then we
526 # only need the support libraries copied
527 ff, enabled = _check_binfmt_misc(args.executable)
528
529 if not enabled:
530 print("binfmt_misc not enabled, update disabled")
531 return 1
532
533 if ff:
534 tmp_tar.add(args.executable, arcname=ff)
535
536 # Add any associated libraries
537 libs = _get_so_libs(args.executable)
538 if libs:
539 for l in libs:
540 tmp_tar.add(os.path.realpath(l), arcname=l)
541
542 # Create a Docker buildfile
543 df = StringIO()
544 df.write(u"FROM %s\n" % args.tag)
545 df.write(u"ADD . /\n")
546
547 df_bytes = BytesIO(bytes(df.getvalue(), "UTF-8"))
548
549 df_tar = TarInfo(name="Dockerfile")
550 df_tar.size = df_bytes.getbuffer().nbytes
551 tmp_tar.addfile(df_tar, fileobj=df_bytes)
552
553 tmp_tar.close()
554
555 # reset the file pointers
556 tmp.flush()
557 tmp.seek(0)
558
559 # Run the build with our tarball context
560 dkr = Docker()
561 dkr.update_image(args.tag, tmp, quiet=args.quiet)
562
563 return 0
564
565
566 class CleanCommand(SubCommand):
567 """Clean up docker instances"""
568 name = "clean"
569
570 def run(self, args, argv):
571 Docker().clean()
572 return 0
573
574
575 class ImagesCommand(SubCommand):
576 """Run "docker images" command"""
577 name = "images"
578
579 def run(self, args, argv):
580 return Docker().command("images", argv, args.quiet)
581
582
583 class ProbeCommand(SubCommand):
584 """Probe if we can run docker automatically"""
585 name = "probe"
586
587 def run(self, args, argv):
588 try:
589 docker = Docker()
590 if docker._command[0] == "docker":
591 print("docker")
592 elif docker._command[0] == "sudo":
593 print("sudo docker")
594 elif docker._command[0] == "podman":
595 print("podman")
596 except Exception:
597 print("no")
598
599 return
600
601
602 class CcCommand(SubCommand):
603 """Compile sources with cc in images"""
604 name = "cc"
605
606 def args(self, parser):
607 parser.add_argument("--image", "-i", required=True,
608 help="The docker image in which to run cc")
609 parser.add_argument("--cc", default="cc",
610 help="The compiler executable to call")
611 parser.add_argument("--source-path", "-s", nargs="*", dest="paths",
612 help="""Extra paths to (ro) mount into container for
613 reading sources""")
614
615 def run(self, args, argv):
616 if argv and argv[0] == "--":
617 argv = argv[1:]
618 cwd = os.getcwd()
619 cmd = ["--rm", "-w", cwd,
620 "-v", "%s:%s:rw" % (cwd, cwd)]
621 if args.paths:
622 for p in args.paths:
623 cmd += ["-v", "%s:%s:ro,z" % (p, p)]
624 cmd += [args.image, args.cc]
625 cmd += argv
626 return Docker().run(cmd, False, quiet=args.quiet,
627 as_user=True)
628
629
630 class CheckCommand(SubCommand):
631 """Check if we need to re-build a docker image out of a dockerfile.
632 Arguments: <tag> <dockerfile>"""
633 name = "check"
634
635 def args(self, parser):
636 parser.add_argument("tag",
637 help="Image Tag")
638 parser.add_argument("dockerfile", default=None,
639 help="Dockerfile name", nargs='?')
640 parser.add_argument("--checktype", choices=["checksum", "age"],
641 default="checksum", help="check type")
642 parser.add_argument("--olderthan", default=60, type=int,
643 help="number of minutes")
644
645 def run(self, args, argv):
646 tag = args.tag
647
648 try:
649 dkr = Docker()
650 except subprocess.CalledProcessError:
651 print("Docker not set up")
652 return 1
653
654 info = dkr.inspect_tag(tag)
655 if info is None:
656 print("Image does not exist")
657 return 1
658
659 if args.checktype == "checksum":
660 if not args.dockerfile:
661 print("Need a dockerfile for tag:%s" % (tag))
662 return 1
663
664 dockerfile = _read_dockerfile(args.dockerfile)
665
666 if dkr.image_matches_dockerfile(tag, dockerfile):
667 if not args.quiet:
668 print("Image is up to date")
669 return 0
670 else:
671 print("Image needs updating")
672 return 1
673 elif args.checktype == "age":
674 timestr = dkr.get_image_creation_time(info).split(".")[0]
675 created = datetime.strptime(timestr, "%Y-%m-%dT%H:%M:%S")
676 past = datetime.now() - timedelta(minutes=args.olderthan)
677 if created < past:
678 print ("Image created @ %s more than %d minutes old" %
679 (timestr, args.olderthan))
680 return 1
681 else:
682 if not args.quiet:
683 print ("Image less than %d minutes old" % (args.olderthan))
684 return 0
685
686
687 def main():
688 global USE_ENGINE
689
690 parser = argparse.ArgumentParser(description="A Docker helper",
691 usage="%s <subcommand> ..." %
692 os.path.basename(sys.argv[0]))
693 parser.add_argument("--engine", type=EngineEnum.argparse, choices=list(EngineEnum),
694 help="specify which container engine to use")
695 subparsers = parser.add_subparsers(title="subcommands", help=None)
696 for cls in SubCommand.__subclasses__():
697 cmd = cls()
698 subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
699 cmd.shared_args(subp)
700 cmd.args(subp)
701 subp.set_defaults(cmdobj=cmd)
702 args, argv = parser.parse_known_args()
703 if args.engine:
704 USE_ENGINE = args.engine
705 return args.cmdobj.run(args, argv)
706
707
708 if __name__ == "__main__":
709 sys.exit(main())