1 """This module contains the main part of Instant, the build_module function."""
2
3 import os, sys, shutil, glob, errno
4 from itertools import chain
5
6
7 from output import *
8 from config import header_and_libs_from_pkgconfig, get_swig_version
9 from paths import *
10 from signatures import *
11 from cache import *
12 from codegeneration import *
13 from locking import get_lock, release_lock
14
16 instant_assert(isinstance(x, str),
17 "In instant.build_module: Expecting string.")
18
20 instant_assert(isinstance(x, bool),
21 "In instant.build_module: Expecting bool.")
22
24 instant_assert(isinstance(x, (list, tuple)),
25 "In instant.build_module: Expecting sequence.")
26 instant_assert(all(isinstance(i, str) for i in x),
27 "In instant.build_module: Expecting sequence of strings.")
28
32
34 if isinstance(x, str):
35 x = x.split()
36 return strip_strings(x)
37
39 """
40 Creates a directory (tree). If directory already excists it does nothing.
41 """
42 try:
43 os.makedirs(path)
44 except os.error, e:
45 if e.errno != errno.EEXIST:
46 raise
47
49 """Copy a list of files from a source directory to a destination directory.
50 This may seem a bit complicated, but a lot of this code is error checking."""
51 if os.path.exists(dest):
52 overwriting = set(files) & set(glob.glob(os.path.join(dest, "*")))
53 if overwriting:
54 instant_warning("In instant.copy_files: Path '%s' already exists, "\
55 "overwriting existing files: %r." % (dest, list(overwriting)))
56 else:
57 makedirs(dest)
58
59 if source != dest:
60 instant_debug("In instant.copy_files: Copying files %r from %r to %r"\
61 % (files, source, dest))
62
63 for f in files:
64 a = os.path.join(source, f)
65 b = os.path.join(dest, f)
66 instant_assert(a != b, "In instant.copy_files: Seems like the "\
67 "input files are absolute paths, should be relative to "\
68 "source. (%r, %r)" % (a, b))
69 instant_assert(os.path.isfile(a), "In instant.copy_files: "\
70 "Missing source file '%s'." % a)
71 if os.path.isfile(b):
72 os.remove(b)
73 shutil.copyfile(a, b)
74
75
76 -def recompile(modulename, module_path, new_compilation_checksum,
77 build_system="distutils"):
78 """Recompile module if the new checksum is different from
79 the one in the checksum file in the module directory."""
80
81 assert(build_system in ["distutils", "cmake"])
82
83 compilation_checksum_filename = "%s.checksum" % modulename
84 if os.path.exists(compilation_checksum_filename):
85 checksum_file = open(compilation_checksum_filename)
86 old_compilation_checksum = checksum_file.readline()
87 checksum_file.close()
88 if old_compilation_checksum == new_compilation_checksum:
89 return
90
91
92 try:
93 get_swig_version()
94 except OSError:
95 instant_error("In instant.recompile: Could not find swig!"\
96 " You can download swig from http://www.swig.org")
97
98
99 compile_log_filename = os.path.join(module_path, "compile.log")
100 compile_log_filename_dest = os.path.join(get_default_error_dir(), \
101 modulename, "compile.log")
102 compile_log_file = open(compile_log_filename, "w")
103
104 ret = 1
105 try:
106 compile_log_contents = None
107 instant_info("--- Instant: compiling ---")
108
109
110
111 if build_system == "distutils":
112
113 cmd = "python setup.py build_ext install --install-platlib=."
114 instant_debug("cmd = %s" % cmd)
115 ret, output = get_status_output(cmd)
116 compile_log_file.write(output)
117 compile_log_file.flush()
118 if ret != 0:
119 compile_log_contents = output
120 if os.path.exists(compilation_checksum_filename):
121 os.remove(compilation_checksum_filename)
122 msg = "In instant.recompile: The module did not compile with command '%s', see '%s'"
123 instant_error(msg % (cmd, compile_log_filename_dest))
124
125 else:
126
127 cmd = "cmake -DDEBUG=TRUE . ";
128 instant_debug("cmd = %s" % cmd)
129 ret, output = get_status_output(cmd)
130 compile_log_file.write(output)
131 compile_log_file.flush()
132 if ret != 0:
133 compile_log_contents = output
134 if os.path.exists(compilation_checksum_filename):
135 os.remove(compilation_checksum_filename)
136 msg = "In instant.recompile: The module did not compile with command '%s', see '%s'"
137 instant_error(msg % (cmd, compile_log_filename_dest))
138
139
140 cmd = "make VERBOSE=1 "
141 instant_debug("cmd = %s" % cmd)
142 ret, output = get_status_output(cmd)
143 compile_log_file.write(output)
144 compile_log_file.flush()
145 if ret != 0:
146 compile_log_contents = output
147 if os.path.exists(compilation_checksum_filename):
148 os.remove(compilation_checksum_filename)
149 msg = "In instant.recompile: The module did not compile with command '%s', see '%s'"
150 instant_error(msg % (cmd, compile_log_filename_dest))
151
152 finally:
153 compile_log_file.close()
154 if ret != 0:
155 if "INSTANT_DISPLAY_COMPILE_LOG" in os.environ.keys():
156 instant_warning("")
157 instant_warning("Content of instant compile.log")
158 instant_warning("==============================")
159 instant_warning(compile_log_contents)
160 instant_warning("")
161
162
163 module_path = copy_to_cache(module_path, get_default_error_dir(),
164 modulename, check_for_existing_path=False)
165
166
167 write_file(compilation_checksum_filename, new_compilation_checksum)
168
169
170 -def copy_to_cache(module_path, cache_dir, modulename, \
171 check_for_existing_path=True):
172 "Copy module directory to cache."
173
174
175
176 lock = get_lock(cache_dir, modulename)
177
178
179 cache_module_path = os.path.join(cache_dir, modulename)
180 if check_for_existing_path and os.path.exists(cache_module_path):
181
182 instant_warning("In instant.build_module: Path '%s' already exists,"\
183 " but module wasn't found in cache previously. Not overwriting,"\
184 " assuming this module is valid." % cache_module_path)
185
186 release_lock(lock)
187 return cache_module_path
188
189
190
191
192
193 instant_assert(os.path.isdir(module_path), "In instant.build_module:"\
194 " Cannot copy non-existing directory %r!" % module_path)
195 if check_for_existing_path and os.path.isdir(cache_module_path):
196 instant_error("In instant.build_module: Cache directory %r shouldn't"\
197 " exist at this point!" % cache_module_path)
198 instant_debug("In instant.build_module: Copying built module from %r"\
199 " to cache at %r" % (module_path, cache_module_path))
200
201
202 try:
203 shutil.copytree(module_path, cache_module_path)
204 except OSError, e:
205 if e.errno != errno.EEXIST:
206 raise
207 finally:
208 delete_temp_dir()
209 release_lock(lock)
210
211 return cache_module_path
212
213 -def build_module(modulename=None, source_directory=".",
214 code="", init_code="",
215 additional_definitions="", additional_declarations="",
216 sources=[], wrap_headers=[],
217 local_headers=[], system_headers=[],
218 include_dirs=['.'], library_dirs=[], libraries=[],
219 swigargs=['-c++', '-fcompact', '-O', '-I.', '-small'],
220 swig_include_dirs = [],
221 cppargs=['-O2'], lddargs=[],
222 object_files=[], arrays=[],
223 generate_interface=True, generate_setup=True,
224 cmake_packages=[],
225 signature=None, cache_dir=None):
226 """Generate and compile a module from C/C++ code using SWIG.
227
228 Arguments:
229 ==========
230 The keyword arguments are as follows:
231 - B{modulename}:
232 - The name you want for the module.
233 If specified, the module will not be cached.
234 If missing, a name will be constructed based on
235 a checksum of the other arguments, and the module
236 will be placed in the global cache. String.
237 - B{source_directory}:
238 - The directory where user supplied files reside. The files
239 given in B{sources}, B{wrap_headers}, and B{local_headers}
240 are expected to exist in this directory. String.
241 - B{code}:
242 - A string containing C or C++ code to be compiled and wrapped. String.
243 - B{init_code}:
244 - Code that should be executed when the Instant module is
245 imported. This code is inserted in the SWIG interface file, and is
246 used for instance for calling C{import_array()} used for the
247 initialization of NumPy arrays. String.
248 - B{additional_definitions}:
249 - Additional definitions (typically needed for inheritance)
250 for interface file. These definitions should be given as triple-quoted
251 strings in the case they span multiple lines, and are placed both in the
252 initial block for C/C++ code (C{%{,%}}-block), and the main section
253 of the interface file. String.
254 - B{additional_declarations}:
255 - Additional declarations (typically needed for inheritance)
256 for interface file. These declarations should be given as triple-quoted
257 strings in the case they span multiple lines, and are plaves in the main
258 section of the interface file. String.
259 - B{sources}:
260 - Source files to compile and link with the module. These
261 files are compiled togehter with the SWIG-generated wrapper file into
262 the final library file. Should reside in directory specified in
263 B{source_directory}. List of strings.
264 - B{wrap_headers}:
265 - Local header files that should be wrapped by SWIG. The
266 files specified will be included both in the initial block for C/C++ code
267 (with a C directive) and in the main section of the interface file (with
268 a SWIG directive). Should reside in directory specified in
269 B{source_directory}. List of strings.
270 - B{local_headers}:
271 - Local header files required to compile the wrapped
272 code. The files specified will be included in the initial block for
273 C/C++ code (with a C directive). Should reside in directory specified in
274 B{source_directory}. List of strings.
275 - B{system_headers}:
276 - System header files required to compile the wrapped
277 code. The files specified will be included in the initial block for C/C++
278 code (with a C directive). List of strings.
279 - B{include_dirs}:
280 - Directories to search for header files for building the
281 extension module. Needs to be absolute path names. List of strings.
282 - B{library_dirs}:
283 - Directories to search for libraries (C{-l}) for building
284 the extension module. Needs to be absolute paths. List of strings.
285 - B{libraries}:
286 - Libraries needed by the Instant module. The libraries will
287 be linked in from the shared object file. The initial C{-l} is added
288 automatically. List of strings.
289 - B{swigargs}:
290 - List of arguments to swig, e.g. C{["-lpointers.i"]}
291 to include the SWIG pointers.i library.
292 - B{swig_include_dirs}:
293 - A list of directories to include in the 'swig' command.
294 - B{cppargs}:
295 - List of arguments to the compiler, e.g. C{["-Wall", "-fopenmp"]}.
296 - B{lddargs}:
297 - List of arguments to the linker, e.g. C{["-E", "-U"]}.
298 - B{object_files}:
299 - If you want to compile the files yourself. TODO: Not yet supported.
300 - B{arrays}:
301 - A nested list describing the C arrays to be made from NumPy arrays.
302 The SWIG interface for fil NumPy is used. For 1D arrays, the inner
303 list should contain strings with the variable names for the length of
304 the arrays and the array itself. 2D matrices should contain the names
305 of the dimensions in the two directions as well as the name of the
306 array, and 3D tensors should contain the names of the dimensions in
307 the three directions in addition to the name of the array.
308 If the NumPy array har more than four dimensions, the inner list should
309 contain strings with variable names for the number of dimensions,
310 the length in each dimension as a pointer, and the array itself, respectively.
311 - B{generate_interface}:
312 - A bool to indicate if you want to generate the interface files.
313 - B{generate_setup}:
314 - A bool to indicate if you want to generate the setup.py file.
315 - B{cmake_packages}:
316 - A list with CMake configured packages which are used to configure
317 and build the extension module. If used it will override the default
318 behaviour of using distutils.
319 - B{signature}:
320 - A signature string to identify the form instead of the source code.
321 - B{cache_dir}:
322 - A directory to look for cached modules and place new ones.
323 If missing, a default directory is used. Note that the module
324 will not be cached if B{modulename} is specified.
325 The cache directory should not be used for anything else.
326 """
327
328
329 original_path = os.getcwd()
330
331
332
333 instant_assert(modulename is None or isinstance(modulename, str),
334 "In instant.build_module: Expecting modulename to be string or None.")
335 assert_is_str(source_directory)
336 source_directory = os.path.abspath(source_directory)
337 assert_is_str(code)
338 assert_is_str(init_code)
339 assert_is_str(additional_definitions)
340 assert_is_str(additional_declarations)
341 sources = strip_strings(sources)
342 wrap_headers = strip_strings(wrap_headers)
343 local_headers = strip_strings(local_headers)
344 system_headers = strip_strings(system_headers)
345 include_dirs = strip_strings(include_dirs)
346 library_dirs = strip_strings(library_dirs)
347 libraries = strip_strings(libraries)
348 swigargs = arg_strings(swigargs)
349 swig_include_dirs = strip_strings(swig_include_dirs)
350 cppargs = arg_strings(cppargs)
351 lddargs = arg_strings(lddargs)
352 object_files = strip_strings(object_files)
353 arrays = [strip_strings(a) for a in arrays]
354 assert_is_bool(generate_interface)
355 assert_is_bool(generate_setup)
356 cmake_packages = strip_strings(cmake_packages)
357 instant_assert( signature is None \
358 or isinstance(signature, str) \
359 or hasattr(signature, "signature"),
360 "In instant.build_module: Expecting modulename to be string or None.")
361 instant_assert(not (signature is not None and modulename is not None),
362 "In instant.build_module: Can't have both modulename and signature.")
363
364
365
366 cache_dir = validate_cache_dir(cache_dir)
367
368
369 csrcs = [f for f in sources if f.endswith('.c') or f.endswith('.C')]
370 cppsrcs = [f for f in sources if f.endswith('.cpp') or f.endswith('.cxx')]
371 instant_assert(len(csrcs) + len(cppsrcs) == len(sources),
372 "In instant.build_module: Source files must have '.c' or '.cpp' suffix")
373
374
375 instant_debug('In instant.build_module:')
376 instant_debug('::: Begin Arguments :::')
377 instant_debug(' modulename: %r' % modulename)
378 instant_debug(' source_directory: %r' % source_directory)
379 instant_debug(' code: %r' % code)
380 instant_debug(' init_code: %r' % init_code)
381 instant_debug(' additional_definitions: %r' % additional_definitions)
382 instant_debug(' additional_declarations: %r' % additional_declarations)
383 instant_debug(' sources: %r' % sources)
384 instant_debug(' csrcs: %r' % csrcs)
385 instant_debug(' cppsrcs: %r' % cppsrcs)
386 instant_debug(' wrap_headers: %r' % wrap_headers)
387 instant_debug(' local_headers: %r' % local_headers)
388 instant_debug(' system_headers: %r' % system_headers)
389 instant_debug(' include_dirs: %r' % include_dirs)
390 instant_debug(' library_dirs: %r' % library_dirs)
391 instant_debug(' libraries: %r' % libraries)
392 instant_debug(' swigargs: %r' % swigargs)
393 instant_debug(' swig_include_dirs: %r' % swig_include_dirs)
394 instant_debug(' cppargs: %r' % cppargs)
395 instant_debug(' lddargs: %r' % lddargs)
396 instant_debug(' object_files: %r' % object_files)
397 instant_debug(' arrays: %r' % arrays)
398 instant_debug(' generate_interface: %r' % generate_interface)
399 instant_debug(' generate_setup: %r' % generate_setup)
400 instant_debug(' cmake_packages: %r' % cmake_packages)
401 instant_debug(' signature: %r' % signature)
402 instant_debug(' cache_dir: %r' % cache_dir)
403 instant_debug('::: End Arguments :::')
404
405
406
407
408
409 if modulename is None:
410
411 if signature is None:
412
413
414
415 checksum_args = ( \
416
417
418
419
420 code, init_code,
421 additional_definitions,
422 additional_declarations,
423
424
425
426 system_headers,
427 include_dirs, library_dirs, libraries,
428 swig_include_dirs, swigargs, cppargs, lddargs,
429 object_files, arrays,
430 generate_interface, generate_setup, cmake_packages,
431
432
433 )
434 allfiles = sources + wrap_headers + local_headers
435 allfiles = [os.path.join(source_directory, f) for f in allfiles]
436 text = "\n".join((str(a) for a in checksum_args))
437 signature = modulename_from_checksum(compute_checksum(text, allfiles))
438 modulename = signature
439 moduleids = [signature]
440 else:
441 module, moduleids = check_memory_cache(signature)
442 if module: return module
443 modulename = moduleids[-1]
444
445
446 module = check_disk_cache(modulename, cache_dir, moduleids)
447 if module: return module
448
449
450 module_path = os.path.join(get_temp_dir(), modulename)
451 instant_assert(not os.path.exists(module_path),
452 "In instant.build_module: Not expecting module_path to exist: '%s'"\
453 % module_path)
454 makedirs(module_path)
455 use_cache = True
456 else:
457 use_cache = False
458 moduleids = []
459 module_path = os.path.join(original_path, modulename)
460 makedirs(module_path)
461
462
463
464
465
466
467
468
469
470
471
472 try:
473
474
475 module_path = os.path.abspath(module_path)
476 files_to_copy = sources + wrap_headers + local_headers + object_files
477 copy_files(source_directory, module_path, files_to_copy)
478
479
480
481 os.chdir(module_path)
482
483
484 write_file("__init__.py", "from %s import *" % modulename)
485
486
487 ifile_name = "%s.i" % modulename
488 if generate_interface:
489 write_interfacefile(ifile_name, modulename, code, init_code,
490 additional_definitions, additional_declarations, system_headers,
491 local_headers, wrap_headers, arrays)
492
493
494 if generate_setup and not cmake_packages:
495 setup_name = "setup.py"
496 write_setup(setup_name, modulename, csrcs, cppsrcs, local_headers, \
497 include_dirs, library_dirs, libraries, swig_include_dirs, \
498 swigargs, cppargs, lddargs)
499 build_system = "distutils"
500
501 else:
502 write_cmakefile(modulename, cmake_packages, csrcs, cppsrcs, local_headers, \
503 include_dirs, library_dirs, libraries, swig_include_dirs, \
504 swigargs, cppargs, lddargs)
505 build_system = "cmake"
506
507
508
509
510
511
512
513
514
515
516
517
518 checksum_args = ( \
519
520
521
522
523
524
525
526
527
528
529 system_headers,
530 include_dirs, library_dirs, libraries,
531 swigargs, swig_include_dirs, cppargs, lddargs,
532 object_files,
533
534
535
536
537 )
538 text = "\n".join((str(a) for a in checksum_args))
539 allfiles = sources + wrap_headers + local_headers + [ifile_name]
540 new_compilation_checksum = compute_checksum(text, allfiles)
541
542
543 recompile(modulename, module_path, new_compilation_checksum, build_system)
544
545
546
547
548 if use_cache:
549 module_path = copy_to_cache(module_path, cache_dir, modulename)
550
551
552
553 if use_cache:
554 lock = get_lock(cache_dir, modulename)
555 module = import_and_cache_module(module_path, modulename, moduleids)
556 if use_cache:
557 release_lock(lock)
558 if not module:
559 instant_error("Failed to import newly compiled module!")
560
561 instant_debug("In instant.build_module: Returning %s from build_module."\
562 % module)
563 return module
564
565
566 finally:
567
568 os.chdir(original_path)
569
570 instant_error("In instant.build_module: Should never reach this point!")
571
572
573
575 original_path = os.getcwd()
576 cache_dir = validate_cache_dir(cache_dir)
577 signature = modulename_from_checksum(compute_checksum(c_code))
578 modulename = signature
579 moduleids = [signature]
580 module_path = os.path.join(get_temp_dir(), modulename)
581
582
583 makedirs(module_path)
584 os.chdir(module_path)
585
586 write_cmakefile(modulename)
587 s = generate_interface_file_vtk(signature, c_code)
588 write_vtk_interface_file(signature, c_code)
589
590 ret, output = get_status_output("cmake -DDEBUG=TRUE . > cmake.log ")
591 ret, output = get_status_output("make > compile.log ")
592
593 module_path = copy_to_cache(module_path, cache_dir, modulename)
594
595 os.chdir(original_path)
596 lock = get_lock(cache_dir, modulename)
597
598 module = import_and_cache_module(module_path, modulename, moduleids)
599 release_lock(lock)
600
601 return module
602
604 original_path = os.getcwd()
605 cache_dir = validate_cache_dir(cache_dir)
606 signature = modulename_from_checksum(compute_checksum(c_code))
607 modulename = signature
608 moduleids = [signature]
609 module_path = os.path.join(get_temp_dir(), modulename)
610
611 makedirs(module_path)
612 os.chdir(module_path)
613
614 write_vmtk_cmakefile(modulename)
615 s = generate_interface_file_vtk(signature, c_code)
616 write_vtk_interface_file(signature, c_code)
617
618 ret, output = get_status_output("cmake -DDEBUG=TRUE . > cmake.log ")
619 ret, output = get_status_output("make > compile.log ")
620
621 module_path = copy_to_cache(module_path, cache_dir, modulename)
622
623 os.chdir(original_path)
624 lock = get_lock(cache_dir, modulename)
625
626 module = import_and_cache_module(module_path, modulename, moduleids)
627 release_lock(lock)
628
629 return module
630