citadel/scripts/pythondeps

#!/usr/bin/env python3
#
# Determine dependencies of python scripts or available python modules in a search path.
#
# Given the -d argument and a filename/filenames, returns the modules imported by those files.
# Given the -d argument and a directory/directories, recurses to find all
# python packages and modules, returns the modules imported by these.
# Given the -p argument and a path or paths, scans that path for available python modules/packages.

import argparse
import ast
import imp
import logging
import os.path
import sys


logger = logging.getLogger('pythondeps')

suffixes = []
for triple in imp.get_suffixes():
    suffixes.append(triple[0])


class PythonDepError(Exception):
    pass


class DependError(PythonDepError):
    def __init__(self, path, error):
        self.path = path
        self.error = error
        PythonDepError.__init__(self, error)

    def __str__(self):
        return "Failure determining dependencies of {}: {}".format(self.path, self.error)


class ImportVisitor(ast.NodeVisitor):
    def __init__(self):
        self.imports = set()
        self.importsfrom = []

    def visit_Import(self, node):
        for alias in node.names:
            self.imports.add(alias.name)

    def visit_ImportFrom(self, node):
        self.importsfrom.append((node.module, [a.name for a in node.names], node.level))


def walk_up(path):
    while path:
        yield path
        path, _, _ = path.rpartition(os.sep)


def get_provides(path):
    path = os.path.realpath(path)

    def get_fn_name(fn):
        for suffix in suffixes:
            if fn.endswith(suffix):
                return fn[:-len(suffix)]

    isdir = os.path.isdir(path)
    if isdir:
        pkg_path = path
        walk_path = path
    else:
        pkg_path = get_fn_name(path)
        if pkg_path is None:
            return
        walk_path = os.path.dirname(path)

    for curpath in walk_up(walk_path):
        if not os.path.exists(os.path.join(curpath, '__init__.py')):
            libdir = curpath
            break
    else:
        libdir = ''

    package_relpath = pkg_path[len(libdir)+1:]
    package = '.'.join(package_relpath.split(os.sep))
    if not isdir:
        yield package, path
    else:
        if os.path.exists(os.path.join(path, '__init__.py')):
            yield package, path

        for dirpath, dirnames, filenames in os.walk(path):
            relpath = dirpath[len(path)+1:]
            if relpath:
                if '__init__.py' not in filenames:
                    dirnames[:] = []
                    continue
                else:
                    context = '.'.join(relpath.split(os.sep))
                    if package:
                        context = package + '.' + context
                    yield context, dirpath
            else:
                context = package

            for fn in filenames:
                adjusted_fn = get_fn_name(fn)
                if not adjusted_fn or adjusted_fn == '__init__':
                    continue

                fullfn = os.path.join(dirpath, fn)
                if context:
                    yield context + '.' + adjusted_fn, fullfn
                else:
                    yield adjusted_fn, fullfn


def get_code_depends(code_string, path=None, provide=None, ispkg=False):
    try:
        code = ast.parse(code_string, path)
    except TypeError as exc:
        raise DependError(path, exc)
    except SyntaxError as exc:
        raise DependError(path, exc)

    visitor = ImportVisitor()
    visitor.visit(code)
    for builtin_module in sys.builtin_module_names:
        if builtin_module in visitor.imports:
            visitor.imports.remove(builtin_module)

    if provide:
        provide_elements = provide.split('.')
        if ispkg:
            provide_elements.append("__self__")
        context = '.'.join(provide_elements[:-1])
        package_path = os.path.dirname(path)
    else:
        context = None
        package_path = None

    levelzero_importsfrom = (module for module, names, level in visitor.importsfrom
                             if level == 0)
    for module in visitor.imports | set(levelzero_importsfrom):
        if context and path:
            module_basepath = os.path.join(package_path, module.replace('.', '/'))
            if os.path.exists(module_basepath):
                # Implicit relative import
                yield context + '.' + module, path
                continue

            for suffix in suffixes:
                if os.path.exists(module_basepath + suffix):
                    # Implicit relative import
                    yield context + '.' + module, path
                    break
            else:
                yield module, path
        else:
            yield module, path

    for module, names, level in visitor.importsfrom:
        if level == 0:
            continue
        elif not provide:
            raise DependError("Error: ImportFrom non-zero level outside of a package: {0}".format((module, names, level)), path)
        elif level > len(provide_elements):
            raise DependError("Error: ImportFrom level exceeds package depth: {0}".format((module, names, level)), path)
        else:
            context = '.'.join(provide_elements[:-level])
            if module:
                if context:
                    yield context + '.' + module, path
                else:
                    yield module, path


def get_file_depends(path):
    try:
        code_string = open(path, 'r').read()
    except (OSError, IOError) as exc:
        raise DependError(path, exc)

    return get_code_depends(code_string, path)


def get_depends_recursive(directory):
    directory = os.path.realpath(directory)

    provides = dict((v, k) for k, v in get_provides(directory))
    for filename, provide in provides.items():
        if os.path.isdir(filename):
            filename = os.path.join(filename, '__init__.py')
            ispkg = True
        elif not filename.endswith('.py'):
            continue
        else:
            ispkg = False

        with open(filename, 'r') as f:
            source = f.read()

        depends = get_code_depends(source, filename, provide, ispkg)
        for depend, by in depends:
            yield depend, by


def get_depends(path):
    if os.path.isdir(path):
        return get_depends_recursive(path)
    else:
        return get_file_depends(path)


def main():
    logging.basicConfig()

    parser = argparse.ArgumentParser(description='Determine dependencies and provided packages for python scripts/modules')
    parser.add_argument('path', nargs='+', help='full path to content to be processed')
    group = parser.add_mutually_exclusive_group()
    group.add_argument('-p', '--provides', action='store_true',
                       help='given a path, display the provided python modules')
    group.add_argument('-d', '--depends', action='store_true',
                       help='given a filename, display the imported python modules')

    args = parser.parse_args()
    if args.provides:
        modules = set()
        for path in args.path:
            for provide, fn in get_provides(path):
                modules.add(provide)

        for module in sorted(modules):
            print(module)
    elif args.depends:
        for path in args.path:
            try:
                modules = get_depends(path)
            except PythonDepError as exc:
                logger.error(str(exc))
                sys.exit(1)

            for module, imp_by in modules:
                print("{}\t{}".format(module, imp_by))
    else:
        parser.print_help()
        sys.exit(2)


if __name__ == '__main__':
    main()