1368 lines
49 KiB
Python
1368 lines
49 KiB
Python
|
# interp.py - shell interpreter for pysh.
|
||
|
|
#
|
||
|
|
# Copyright 2007 Patrick Mezard
|
||
|
|
#
|
||
|
|
# This software may be used and distributed according to the terms
|
||
|
|
# of the GNU General Public License, incorporated herein by reference.
|
||
|
|
|
||
|
|
"""Implement the shell interpreter.
|
||
|
|
|
||
|
|
Most references are made to "The Open Group Base Specifications Issue 6".
|
||
|
|
<http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html>
|
||
|
|
"""
|
||
|
|
# TODO: document the fact input streams must implement fileno() so Popen will work correctly.
|
||
|
|
# it requires non-stdin stream to be implemented as files. Still to be tested...
|
||
|
|
# DOC: pathsep is used in PATH instead of ':'. Clearly, there are path syntax issues here.
|
||
|
|
# TODO: stop command execution upon error.
|
||
|
|
# TODO: sort out the filename/io_number mess. It should be possible to use filenames only.
|
||
|
|
# TODO: review subshell implementation
|
||
|
|
# TODO: test environment cloning for non-special builtins
|
||
|
|
# TODO: set -x should not rebuild commands from tokens, assignments/redirections are lost
|
||
|
|
# TODO: unit test for variable assignment
|
||
|
|
# TODO: test error management wrt error type/utility type
|
||
|
|
# TODO: test for binary output everywhere
|
||
|
|
# BUG: debug-parsing does not pass log file to PLY. Maybe a PLY upgrade is necessary.
|
||
|
|
import base64
|
||
|
|
import cPickle as pickle
|
||
|
|
import errno
|
||
|
|
import glob
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
import tempfile
|
||
|
|
|
||
|
|
try:
|
||
|
|
s = set()
|
||
|
|
del s
|
||
|
|
except NameError:
|
||
|
|
from Set import Set as set
|
||
|
|
|
||
|
|
import builtin
|
||
|
|
from sherrors import *
|
||
|
|
import pyshlex
|
||
|
|
import pyshyacc
|
||
|
|
|
||
|
|
def mappend(func, *args, **kargs):
|
||
|
|
"""Like map but assume func returns a list. Returned lists are merged into
|
||
|
|
a single one.
|
||
|
|
"""
|
||
|
|
return reduce(lambda a,b: a+b, map(func, *args, **kargs), [])
|
||
|
|
|
||
|
|
class FileWrapper:
|
||
|
|
"""File object wrapper to ease debugging.
|
||
|
|
|
||
|
|
Allow mode checking and implement file duplication through a simple
|
||
|
|
reference counting scheme. Not sure the latter is really useful since
|
||
|
|
only real file descriptors can be used.
|
||
|
|
"""
|
||
|
|
def __init__(self, mode, file, close=True):
|
||
|
|
if mode not in ('r', 'w', 'a'):
|
||
|
|
raise IOError('invalid mode: %s' % mode)
|
||
|
|
self._mode = mode
|
||
|
|
self._close = close
|
||
|
|
if isinstance(file, FileWrapper):
|
||
|
|
if file._refcount[0] <= 0:
|
||
|
|
raise IOError(0, 'Error')
|
||
|
|
self._refcount = file._refcount
|
||
|
|
self._refcount[0] += 1
|
||
|
|
self._file = file._file
|
||
|
|
else:
|
||
|
|
self._refcount = [1]
|
||
|
|
self._file = file
|
||
|
|
|
||
|
|
def dup(self):
|
||
|
|
return FileWrapper(self._mode, self, self._close)
|
||
|
|
|
||
|
|
def fileno(self):
|
||
|
|
"""fileno() should be only necessary for input streams."""
|
||
|
|
return self._file.fileno()
|
||
|
|
|
||
|
|
def read(self, size=-1):
|
||
|
|
if self._mode!='r':
|
||
|
|
raise IOError(0, 'Error')
|
||
|
|
return self._file.read(size)
|
||
|
|
|
||
|
|
def readlines(self, *args, **kwargs):
|
||
|
|
return self._file.readlines(*args, **kwargs)
|
||
|
|
|
||
|
|
def write(self, s):
|
||
|
|
if self._mode not in ('w', 'a'):
|
||
|
|
raise IOError(0, 'Error')
|
||
|
|
return self._file.write(s)
|
||
|
|
|
||
|
|
def flush(self):
|
||
|
|
self._file.flush()
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
if not self._refcount:
|
||
|
|
return
|
||
|
|
assert self._refcount[0] > 0
|
||
|
|
|
||
|
|
self._refcount[0] -= 1
|
||
|
|
if self._refcount[0] == 0:
|
||
|
|
self._mode = 'c'
|
||
|
|
if self._close:
|
||
|
|
self._file.close()
|
||
|
|
self._refcount = None
|
||
|
|
|
||
|
|
def mode(self):
|
||
|
|
return self._mode
|
||
|
|
|
||
|
|
def __getattr__(self, name):
|
||
|
|
if name == 'name':
|
||
|
|
self.name = getattr(self._file, name)
|
||
|
|
return self.name
|
||
|
|
else:
|
||
|
|
raise AttributeError(name)
|
||
|
|
|
||
|
|
def __del__(self):
|
||
|
|
self.close()
|
||
|
|
|
||
|
|
|
||
|
|
def win32_open_devnull(mode):
|
||
|
|
return open('NUL', mode)
|
||
|
|
|
||
|
|
|
||
|
|
class Redirections:
|
||
|
|
"""Stores open files and their mapping to pseudo-sh file descriptor.
|
||
|
|
"""
|
||
|
|
# BUG: redirections are not handled correctly: 1>&3 2>&3 3>&4 does
|
||
|
|
# not make 1 to redirect to 4
|
||
|
|
def __init__(self, stdin=None, stdout=None, stderr=None):
|
||
|
|
self._descriptors = {}
|
||
|
|
if stdin is not None:
|
||
|
|
self._add_descriptor(0, stdin)
|
||
|
|
if stdout is not None:
|
||
|
|
self._add_descriptor(1, stdout)
|
||
|
|
if stderr is not None:
|
||
|
|
self._add_descriptor(2, stderr)
|
||
|
|
|
||
|
|
def add_here_document(self, interp, name, content, io_number=None):
|
||
|
|
if io_number is None:
|
||
|
|
io_number = 0
|
||
|
|
|
||
|
|
if name==pyshlex.unquote_wordtree(name):
|
||
|
|
content = interp.expand_here_document(('TOKEN', content))
|
||
|
|
|
||
|
|
# Write document content in a temporary file
|
||
|
|
tmp = tempfile.TemporaryFile()
|
||
|
|
try:
|
||
|
|
tmp.write(content)
|
||
|
|
tmp.flush()
|
||
|
|
tmp.seek(0)
|
||
|
|
self._add_descriptor(io_number, FileWrapper('r', tmp))
|
||
|
|
except:
|
||
|
|
tmp.close()
|
||
|
|
raise
|
||
|
|
|
||
|
|
def add(self, interp, op, filename, io_number=None):
|
||
|
|
if op not in ('<', '>', '>|', '>>', '>&'):
|
||
|
|
# TODO: add descriptor duplication and here_documents
|
||
|
|
raise RedirectionError('Unsupported redirection operator "%s"' % op)
|
||
|
|
|
||
|
|
if io_number is not None:
|
||
|
|
io_number = int(io_number)
|
||
|
|
|
||
|
|
if (op == '>&' and filename.isdigit()) or filename=='-':
|
||
|
|
# No expansion for file descriptors, quote them if you want a filename
|
||
|
|
fullname = filename
|
||
|
|
else:
|
||
|
|
if filename.startswith('/'):
|
||
|
|
# TODO: win32 kludge
|
||
|
|
if filename=='/dev/null':
|
||
|
|
fullname = 'NUL'
|
||
|
|
else:
|
||
|
|
# TODO: handle absolute pathnames, they are unlikely to exist on the
|
||
|
|
# current platform (win32 for instance).
|
||
|
|
raise NotImplementedError()
|
||
|
|
else:
|
||
|
|
fullname = interp.expand_redirection(('TOKEN', filename))
|
||
|
|
if not fullname:
|
||
|
|
raise RedirectionError('%s: ambiguous redirect' % filename)
|
||
|
|
# Build absolute path based on PWD
|
||
|
|
fullname = os.path.join(interp.get_env()['PWD'], fullname)
|
||
|
|
|
||
|
|
if op=='<':
|
||
|
|
return self._add_input_redirection(interp, fullname, io_number)
|
||
|
|
elif op in ('>', '>|'):
|
||
|
|
clobber = ('>|'==op)
|
||
|
|
return self._add_output_redirection(interp, fullname, io_number, clobber)
|
||
|
|
elif op=='>>':
|
||
|
|
return self._add_output_appending(interp, fullname, io_number)
|
||
|
|
elif op=='>&':
|
||
|
|
return self._dup_output_descriptor(fullname, io_number)
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
if self._descriptors is not None:
|
||
|
|
for desc in self._descriptors.itervalues():
|
||
|
|
desc.flush()
|
||
|
|
desc.close()
|
||
|
|
self._descriptors = None
|
||
|
|
|
||
|
|
def stdin(self):
|
||
|
|
return self._descriptors[0]
|
||
|
|
|
||
|
|
def stdout(self):
|
||
|
|
return self._descriptors[1]
|
||
|
|
|
||
|
|
def stderr(self):
|
||
|
|
return self._descriptors[2]
|
||
|
|
|
||
|
|
def clone(self):
|
||
|
|
clone = Redirections()
|
||
|
|
for desc, fileobj in self._descriptors.iteritems():
|
||
|
|
clone._descriptors[desc] = fileobj.dup()
|
||
|
|
return clone
|
||
|
|
|
||
|
|
def _add_output_redirection(self, interp, filename, io_number, clobber):
|
||
|
|
if io_number is None:
|
||
|
|
# io_number default to standard output
|
||
|
|
io_number = 1
|
||
|
|
|
||
|
|
if not clobber and interp.get_env().has_opt('-C') and os.path.isfile(filename):
|
||
|
|
# File already exist in no-clobber mode, bail out
|
||
|
|
raise RedirectionError('File "%s" already exists' % filename)
|
||
|
|
|
||
|
|
# Open and register
|
||
|
|
self._add_file_descriptor(io_number, filename, 'w')
|
||
|
|
|
||
|
|
def _add_output_appending(self, interp, filename, io_number):
|
||
|
|
if io_number is None:
|
||
|
|
io_number = 1
|
||
|
|
self._add_file_descriptor(io_number, filename, 'a')
|
||
|
|
|
||
|
|
def _add_input_redirection(self, interp, filename, io_number):
|
||
|
|
if io_number is None:
|
||
|
|
io_number = 0
|
||
|
|
self._add_file_descriptor(io_number, filename, 'r')
|
||
|
|
|
||
|
|
def _add_file_descriptor(self, io_number, filename, mode):
|
||
|
|
try:
|
||
|
|
if filename.startswith('/'):
|
||
|
|
if filename=='/dev/null':
|
||
|
|
f = win32_open_devnull(mode+'b')
|
||
|
|
else:
|
||
|
|
# TODO: handle absolute pathnames, they are unlikely to exist on the
|
||
|
|
# current platform (win32 for instance).
|
||
|
|
raise NotImplementedError('cannot open absolute path %s' % repr(filename))
|
||
|
|
else:
|
||
|
|
f = file(filename, mode+'b')
|
||
|
|
except IOError as e:
|
||
|
|
raise RedirectionError(str(e))
|
||
|
|
|
||
|
|
wrapper = None
|
||
|
|
try:
|
||
|
|
wrapper = FileWrapper(mode, f)
|
||
|
|
f = None
|
||
|
|
self._add_descriptor(io_number, wrapper)
|
||
|
|
except:
|
||
|
|
if f: f.close()
|
||
|
|
if wrapper: wrapper.close()
|
||
|
|
raise
|
||
|
|
|
||
|
|
def _dup_output_descriptor(self, source_fd, dest_fd):
|
||
|
|
if source_fd is None:
|
||
|
|
source_fd = 1
|
||
|
|
self._dup_file_descriptor(source_fd, dest_fd, 'w')
|
||
|
|
|
||
|
|
def _dup_file_descriptor(self, source_fd, dest_fd, mode):
|
||
|
|
source_fd = int(source_fd)
|
||
|
|
if source_fd not in self._descriptors:
|
||
|
|
raise RedirectionError('"%s" is not a valid file descriptor' % str(source_fd))
|
||
|
|
source = self._descriptors[source_fd]
|
||
|
|
|
||
|
|
if source.mode()!=mode:
|
||
|
|
raise RedirectionError('Descriptor %s cannot be duplicated in mode "%s"' % (str(source), mode))
|
||
|
|
|
||
|
|
if dest_fd=='-':
|
||
|
|
# Close the source descriptor
|
||
|
|
del self._descriptors[source_fd]
|
||
|
|
source.close()
|
||
|
|
else:
|
||
|
|
dest_fd = int(dest_fd)
|
||
|
|
if dest_fd not in self._descriptors:
|
||
|
|
raise RedirectionError('Cannot replace file descriptor %s' % str(dest_fd))
|
||
|
|
|
||
|
|
dest = self._descriptors[dest_fd]
|
||
|
|
if dest.mode()!=mode:
|
||
|
|
raise RedirectionError('Descriptor %s cannot be cannot be redirected in mode "%s"' % (str(dest), mode))
|
||
|
|
|
||
|
|
self._descriptors[dest_fd] = source.dup()
|
||
|
|
dest.close()
|
||
|
|
|
||
|
|
def _add_descriptor(self, io_number, file):
|
||
|
|
io_number = int(io_number)
|
||
|
|
|
||
|
|
if io_number in self._descriptors:
|
||
|
|
# Close the current descriptor
|
||
|
|
d = self._descriptors[io_number]
|
||
|
|
del self._descriptors[io_number]
|
||
|
|
d.close()
|
||
|
|
|
||
|
|
self._descriptors[io_number] = file
|
||
|
|
|
||
|
|
def __str__(self):
|
||
|
|
names = [('%d=%r' % (k, getattr(v, 'name', None))) for k,v
|
||
|
|
in self._descriptors.iteritems()]
|
||
|
|
names = ','.join(names)
|
||
|
|
return 'Redirections(%s)' % names
|
||
|
|
|
||
|
|
def __del__(self):
|
||
|
|
self.close()
|
||
|
|
|
||
|
|
def cygwin_to_windows_path(path):
|
||
|
|
"""Turn /cygdrive/c/foo into c:/foo, or return path if it
|
||
|
|
is not a cygwin path.
|
||
|
|
"""
|
||
|
|
if not path.startswith('/cygdrive/'):
|
||
|
|
return path
|
||
|
|
path = path[len('/cygdrive/'):]
|
||
|
|
path = path[:1] + ':' + path[1:]
|
||
|
|
return path
|
||
|
|
|
||
|
|
def win32_to_unix_path(path):
|
||
|
|
if path is not None:
|
||
|
|
path = path.replace('\\', '/')
|
||
|
|
return path
|
||
|
|
|
||
|
|
_RE_SHEBANG = re.compile(r'^\#!\s?([^\s]+)(?:\s([^\s]+))?')
|
||
|
|
_SHEBANG_CMDS = {
|
||
|
|
'/usr/bin/env': 'env',
|
||
|
|
'/bin/sh': 'pysh',
|
||
|
|
'python': 'python',
|
||
|
|
}
|
||
|
|
|
||
|
|
def resolve_shebang(path, ignoreshell=False):
|
||
|
|
"""Return a list of arguments as shebang interpreter call or an empty list
|
||
|
|
if path does not refer to an executable script.
|
||
|
|
See <http://www.opengroup.org/austin/docs/austin_51r2.txt>.
|
||
|
|
|
||
|
|
ignoreshell - set to True to ignore sh shebangs. Return an empty list instead.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
f = file(path)
|
||
|
|
try:
|
||
|
|
# At most 80 characters in the first line
|
||
|
|
header = f.read(80).splitlines()[0]
|
||
|
|
finally:
|
||
|
|
f.close()
|
||
|
|
|
||
|
|
m = _RE_SHEBANG.search(header)
|
||
|
|
if not m:
|
||
|
|
return []
|
||
|
|
cmd, arg = m.group(1,2)
|
||
|
|
if os.path.isfile(cmd):
|
||
|
|
# Keep this one, the hg script for instance contains a weird windows
|
||
|
|
# shebang referencing the current python install.
|
||
|
|
cmdfile = os.path.basename(cmd).lower()
|
||
|
|
if cmdfile == 'python.exe':
|
||
|
|
cmd = 'python'
|
||
|
|
pass
|
||
|
|
elif cmd not in _SHEBANG_CMDS:
|
||
|
|
raise CommandNotFound('Unknown interpreter "%s" referenced in '\
|
||
|
|
'shebang' % header)
|
||
|
|
cmd = _SHEBANG_CMDS.get(cmd)
|
||
|
|
if cmd is None or (ignoreshell and cmd == 'pysh'):
|
||
|
|
return []
|
||
|
|
if arg is None:
|
||
|
|
return [cmd, win32_to_unix_path(path)]
|
||
|
|
return [cmd, arg, win32_to_unix_path(path)]
|
||
|
|
except IOError as e:
|
||
|
|
if e.errno!=errno.ENOENT and \
|
||
|
|
(e.errno!=errno.EPERM and not os.path.isdir(path)): # Opening a directory raises EPERM
|
||
|
|
raise
|
||
|
|
return []
|
||
|
|
|
||
|
|
def win32_find_in_path(name, path):
|
||
|
|
if isinstance(path, str):
|
||
|
|
path = path.split(os.pathsep)
|
||
|
|
|
||
|
|
exts = os.environ.get('PATHEXT', '').lower().split(os.pathsep)
|
||
|
|
for p in path:
|
||
|
|
p_name = os.path.join(p, name)
|
||
|
|
|
||
|
|
prefix = resolve_shebang(p_name)
|
||
|
|
if prefix:
|
||
|
|
return prefix
|
||
|
|
|
||
|
|
for ext in exts:
|
||
|
|
p_name_ext = p_name + ext
|
||
|
|
if os.path.exists(p_name_ext):
|
||
|
|
return [win32_to_unix_path(p_name_ext)]
|
||
|
|
return []
|
||
|
|
|
||
|
|
class Traps(dict):
|
||
|
|
def __setitem__(self, key, value):
|
||
|
|
if key not in ('EXIT',):
|
||
|
|
raise NotImplementedError()
|
||
|
|
super(Traps, self).__setitem__(key, value)
|
||
|
|
|
||
|
|
# IFS white spaces character class
|
||
|
|
_IFS_WHITESPACES = (' ', '\t', '\n')
|
||
|
|
|
||
|
|
class Environment:
|
||
|
|
"""Environment holds environment variables, export table, function
|
||
|
|
definitions and whatever is defined in 2.12 "Shell Execution Environment",
|
||
|
|
redirection excepted.
|
||
|
|
"""
|
||
|
|
def __init__(self, pwd):
|
||
|
|
self._opt = set() #Shell options
|
||
|
|
|
||
|
|
self._functions = {}
|
||
|
|
self._env = {'?': '0', '#': '0'}
|
||
|
|
self._exported = set([
|
||
|
|
'HOME', 'IFS', 'PATH'
|
||
|
|
])
|
||
|
|
|
||
|
|
# Set environment vars with side-effects
|
||
|
|
self._ifs_ws = None # Set of IFS whitespace characters
|
||
|
|
self._ifs_re = None # Regular expression used to split between words using IFS classes
|
||
|
|
self['IFS'] = ''.join(_IFS_WHITESPACES) #Default environment values
|
||
|
|
self['PWD'] = pwd
|
||
|
|
self.traps = Traps()
|
||
|
|
|
||
|
|
def clone(self, subshell=False):
|
||
|
|
env = Environment(self['PWD'])
|
||
|
|
env._opt = set(self._opt)
|
||
|
|
for k,v in self.get_variables().iteritems():
|
||
|
|
if k in self._exported:
|
||
|
|
env.export(k,v)
|
||
|
|
elif subshell:
|
||
|
|
env[k] = v
|
||
|
|
|
||
|
|
if subshell:
|
||
|
|
env._functions = dict(self._functions)
|
||
|
|
|
||
|
|
return env
|
||
|
|
|
||
|
|
def __getitem__(self, key):
|
||
|
|
if key in ('@', '*', '-', '$'):
|
||
|
|
raise NotImplementedError('%s is not implemented' % repr(key))
|
||
|
|
return self._env[key]
|
||
|
|
|
||
|
|
def get(self, key, defval=None):
|
||
|
|
try:
|
||
|
|
return self[key]
|
||
|
|
except KeyError:
|
||
|
|
return defval
|
||
|
|
|
||
|
|
def __setitem__(self, key, value):
|
||
|
|
if key=='IFS':
|
||
|
|
# Update the whitespace/non-whitespace classes
|
||
|
|
self._update_ifs(value)
|
||
|
|
elif key=='PWD':
|
||
|
|
pwd = os.path.abspath(value)
|
||
|
|
if not os.path.isdir(pwd):
|
||
|
|
raise VarAssignmentError('Invalid directory %s' % value)
|
||
|
|
value = pwd
|
||
|
|
elif key in ('?', '!'):
|
||
|
|
value = str(int(value))
|
||
|
|
self._env[key] = value
|
||
|
|
|
||
|
|
def __delitem__(self, key):
|
||
|
|
if key in ('IFS', 'PWD', '?'):
|
||
|
|
raise VarAssignmentError('%s cannot be unset' % key)
|
||
|
|
del self._env[key]
|
||
|
|
|
||
|
|
def __contains__(self, item):
|
||
|
|
return item in self._env
|
||
|
|
|
||
|
|
def set_positional_args(self, args):
|
||
|
|
"""Set the content of 'args' as positional argument from 1 to len(args).
|
||
|
|
Return previous argument as a list of strings.
|
||
|
|
"""
|
||
|
|
# Save and remove previous arguments
|
||
|
|
prevargs = []
|
||
|
|
for i in range(int(self._env['#'])):
|
||
|
|
i = str(i+1)
|
||
|
|
prevargs.append(self._env[i])
|
||
|
|
del self._env[i]
|
||
|
|
self._env['#'] = '0'
|
||
|
|
|
||
|
|
#Set new ones
|
||
|
|
for i,arg in enumerate(args):
|
||
|
|
self._env[str(i+1)] = str(arg)
|
||
|
|
self._env['#'] = str(len(args))
|
||
|
|
|
||
|
|
return prevargs
|
||
|
|
|
||
|
|
def get_positional_args(self):
|
||
|
|
return [self._env[str(i+1)] for i in range(int(self._env['#']))]
|
||
|
|
|
||
|
|
def get_variables(self):
|
||
|
|
return dict(self._env)
|
||
|
|
|
||
|
|
def export(self, key, value=None):
|
||
|
|
if value is not None:
|
||
|
|
self[key] = value
|
||
|
|
self._exported.add(key)
|
||
|
|
|
||
|
|
def get_exported(self):
|
||
|
|
return [(k,self._env.get(k)) for k in self._exported]
|
||
|
|
|
||
|
|
def split_fields(self, word):
|
||
|
|
if not self._ifs_ws or not word:
|
||
|
|
return [word]
|
||
|
|
return re.split(self._ifs_re, word)
|
||
|
|
|
||
|
|
def _update_ifs(self, value):
|
||
|
|
"""Update the split_fields related variables when IFS character set is
|
||
|
|
changed.
|
||
|
|
"""
|
||
|
|
# TODO: handle NULL IFS
|
||
|
|
|
||
|
|
# Separate characters in whitespace and non-whitespace
|
||
|
|
chars = set(value)
|
||
|
|
ws = [c for c in chars if c in _IFS_WHITESPACES]
|
||
|
|
nws = [c for c in chars if c not in _IFS_WHITESPACES]
|
||
|
|
|
||
|
|
# Keep whitespaces in a string for left and right stripping
|
||
|
|
self._ifs_ws = ''.join(ws)
|
||
|
|
|
||
|
|
# Build a regexp to split fields
|
||
|
|
trailing = '[' + ''.join([re.escape(c) for c in ws]) + ']'
|
||
|
|
if nws:
|
||
|
|
# First, the single non-whitespace occurence.
|
||
|
|
nws = '[' + ''.join([re.escape(c) for c in nws]) + ']'
|
||
|
|
nws = '(?:' + trailing + '*' + nws + trailing + '*' + '|' + trailing + '+)'
|
||
|
|
else:
|
||
|
|
# Then mix all parts with quantifiers
|
||
|
|
nws = trailing + '+'
|
||
|
|
self._ifs_re = re.compile(nws)
|
||
|
|
|
||
|
|
def has_opt(self, opt, val=None):
|
||
|
|
return (opt, val) in self._opt
|
||
|
|
|
||
|
|
def set_opt(self, opt, val=None):
|
||
|
|
self._opt.add((opt, val))
|
||
|
|
|
||
|
|
def find_in_path(self, name, pwd=False):
|
||
|
|
path = self._env.get('PATH', '').split(os.pathsep)
|
||
|
|
if pwd:
|
||
|
|
path[:0] = [self['PWD']]
|
||
|
|
if os.name == 'nt':
|
||
|
|
return win32_find_in_path(name, self._env.get('PATH', ''))
|
||
|
|
else:
|
||
|
|
raise NotImplementedError()
|
||
|
|
|
||
|
|
def define_function(self, name, body):
|
||
|
|
if not is_name(name):
|
||
|
|
raise ShellSyntaxError('%s is not a valid function name' % repr(name))
|
||
|
|
self._functions[name] = body
|
||
|
|
|
||
|
|
def remove_function(self, name):
|
||
|
|
del self._functions[name]
|
||
|
|
|
||
|
|
def is_function(self, name):
|
||
|
|
return name in self._functions
|
||
|
|
|
||
|
|
def get_function(self, name):
|
||
|
|
return self._functions.get(name)
|
||
|
|
|
||
|
|
|
||
|
|
name_charset = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
|
||
|
|
name_charset = dict(zip(name_charset,name_charset))
|
||
|
|
|
||
|
|
def match_name(s):
|
||
|
|
"""Return the length in characters of the longest prefix made of name
|
||
|
|
allowed characters in s.
|
||
|
|
"""
|
||
|
|
for i,c in enumerate(s):
|
||
|
|
if c not in name_charset:
|
||
|
|
return s[:i]
|
||
|
|
return s
|
||
|
|
|
||
|
|
def is_name(s):
|
||
|
|
return len([c for c in s if c not in name_charset])<=0
|
||
|
|
|
||
|
|
def is_special_param(c):
|
||
|
|
return len(c)==1 and c in ('@','*','#','?','-','$','!','0')
|
||
|
|
|
||
|
|
def utility_not_implemented(name, *args, **kwargs):
|
||
|
|
raise NotImplementedError('%s utility is not implemented' % name)
|
||
|
|
|
||
|
|
|
||
|
|
class Utility:
|
||
|
|
"""Define utilities properties:
|
||
|
|
func -- utility callable. See builtin module for utility samples.
|
||
|
|
is_special -- see XCU 2.8.
|
||
|
|
"""
|
||
|
|
def __init__(self, func, is_special=0):
|
||
|
|
self.func = func
|
||
|
|
self.is_special = bool(is_special)
|
||
|
|
|
||
|
|
|
||
|
|
def encodeargs(args):
|
||
|
|
def encodearg(s):
|
||
|
|
lines = base64.encodestring(s)
|
||
|
|
lines = [l.splitlines()[0] for l in lines]
|
||
|
|
return ''.join(lines)
|
||
|
|
|
||
|
|
s = pickle.dumps(args)
|
||
|
|
return encodearg(s)
|
||
|
|
|
||
|
|
def decodeargs(s):
|
||
|
|
s = base64.decodestring(s)
|
||
|
|
return pickle.loads(s)
|
||
|
|
|
||
|
|
|
||
|
|
class GlobError(Exception):
|
||
|
|
pass
|
||
|
|
|
||
|
|
class Options:
|
||
|
|
def __init__(self):
|
||
|
|
# True if Mercurial operates with binary streams
|
||
|
|
self.hgbinary = True
|
||
|
|
|
||
|
|
class Interpreter:
|
||
|
|
# Implementation is very basic: the execute() method just makes a DFS on the
|
||
|
|
# AST and execute nodes one by one. Nodes are tuple (name,obj) where name
|
||
|
|
# is a string identifier and obj the AST element returned by the parser.
|
||
|
|
#
|
||
|
|
# Handler are named after the node identifiers.
|
||
|
|
# TODO: check node names and remove the switch in execute with some
|
||
|
|
# dynamic getattr() call to find node handlers.
|
||
|
|
"""Shell interpreter.
|
||
|
|
|
||
|
|
The following debugging flags can be passed:
|
||
|
|
debug-parsing - enable PLY debugging.
|
||
|
|
debug-tree - print the generated AST.
|
||
|
|
debug-cmd - trace command execution before word expansion, plus exit status.
|
||
|
|
debug-utility - trace utility execution.
|
||
|
|
"""
|
||
|
|
|
||
|
|
# List supported commands.
|
||
|
|
COMMANDS = {
|
||
|
|
'cat': Utility(builtin.utility_cat,),
|
||
|
|
'cd': Utility(builtin.utility_cd,),
|
||
|
|
':': Utility(builtin.utility_colon,),
|
||
|
|
'echo': Utility(builtin.utility_echo),
|
||
|
|
'env': Utility(builtin.utility_env),
|
||
|
|
'exit': Utility(builtin.utility_exit),
|
||
|
|
'export': Utility(builtin.builtin_export, is_special=1),
|
||
|
|
'egrep': Utility(builtin.utility_egrep),
|
||
|
|
'fgrep': Utility(builtin.utility_fgrep),
|
||
|
|
'gunzip': Utility(builtin.utility_gunzip),
|
||
|
|
'kill': Utility(builtin.utility_kill),
|
||
|
|
'mkdir': Utility(builtin.utility_mkdir),
|
||
|
|
'netstat': Utility(builtin.utility_netstat),
|
||
|
|
'printf': Utility(builtin.utility_printf),
|
||
|
|
'pwd': Utility(builtin.utility_pwd),
|
||
|
|
'return': Utility(builtin.builtin_return, is_special=1),
|
||
|
|
'sed': Utility(builtin.utility_sed,),
|
||
|
|
'set': Utility(builtin.builtin_set,),
|
||
|
|
'shift': Utility(builtin.builtin_shift,),
|
||
|
|
'sleep': Utility(builtin.utility_sleep,),
|
||
|
|
'sort': Utility(builtin.utility_sort,),
|
||
|
|
'trap': Utility(builtin.builtin_trap, is_special=1),
|
||
|
|
'true': Utility(builtin.utility_true),
|
||
|
|
'unset': Utility(builtin.builtin_unset, is_special=1),
|
||
|
|
'wait': Utility(builtin.builtin_wait, is_special=1),
|
||
|
|
}
|
||
|
|
|
||
|
|
def __init__(self, pwd, debugflags = [], env=None, redirs=None, stdin=None,
|
||
|
|
stdout=None, stderr=None, opts=Options()):
|
||
|
|
self._env = env
|
||
|
|
if self._env is None:
|
||
|
|
self._env = Environment(pwd)
|
||
|
|
self._children = {}
|
||
|
|
|
||
|
|
self._redirs = redirs
|
||
|
|
self._close_redirs = False
|
||
|
|
|
||
|
|
if self._redirs is None:
|
||
|
|
if stdin is None:
|
||
|
|
stdin = sys.stdin
|
||
|
|
if stdout is None:
|
||
|
|
stdout = sys.stdout
|
||
|
|
if stderr is None:
|
||
|
|
stderr = sys.stderr
|
||
|
|
stdin = FileWrapper('r', stdin, False)
|
||
|
|
stdout = FileWrapper('w', stdout, False)
|
||
|
|
stderr = FileWrapper('w', stderr, False)
|
||
|
|
self._redirs = Redirections(stdin, stdout, stderr)
|
||
|
|
self._close_redirs = True
|
||
|
|
|
||
|
|
self._debugflags = list(debugflags)
|
||
|
|
self._logfile = sys.stderr
|
||
|
|
self._options = opts
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
"""Must be called when the interpreter is no longer used."""
|
||
|
|
script = self._env.traps.get('EXIT')
|
||
|
|
if script:
|
||
|
|
try:
|
||
|
|
self.execute_script(script=script)
|
||
|
|
except:
|
||
|
|
pass
|
||
|
|
|
||
|
|
if self._redirs is not None and self._close_redirs:
|
||
|
|
self._redirs.close()
|
||
|
|
self._redirs = None
|
||
|
|
|
||
|
|
def log(self, s):
|
||
|
|
self._logfile.write(s)
|
||
|
|
self._logfile.flush()
|
||
|
|
|
||
|
|
def __getitem__(self, key):
|
||
|
|
return self._env[key]
|
||
|
|
|
||
|
|
def __setitem__(self, key, value):
|
||
|
|
self._env[key] = value
|
||
|
|
|
||
|
|
def options(self):
|
||
|
|
return self._options
|
||
|
|
|
||
|
|
def redirect(self, redirs, ios):
|
||
|
|
def add_redir(io):
|
||
|
|
if isinstance(io, pyshyacc.IORedirect):
|
||
|
|
redirs.add(self, io.op, io.filename, io.io_number)
|
||
|
|
else:
|
||
|
|
redirs.add_here_document(self, io.name, io.content, io.io_number)
|
||
|
|
|
||
|
|
map(add_redir, ios)
|
||
|
|
return redirs
|
||
|
|
|
||
|
|
def execute_script(self, script=None, ast=None, sourced=False,
|
||
|
|
scriptpath=None):
|
||
|
|
"""If script is not None, parse the input. Otherwise takes the supplied
|
||
|
|
AST. Then execute the AST.
|
||
|
|
Return the script exit status.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
if scriptpath is not None:
|
||
|
|
self._env['0'] = os.path.abspath(scriptpath)
|
||
|
|
|
||
|
|
if script is not None:
|
||
|
|
debug_parsing = ('debug-parsing' in self._debugflags)
|
||
|
|
cmds, script = pyshyacc.parse(script, True, debug_parsing)
|
||
|
|
if 'debug-tree' in self._debugflags:
|
||
|
|
pyshyacc.print_commands(cmds, self._logfile)
|
||
|
|
self._logfile.flush()
|
||
|
|
else:
|
||
|
|
cmds, script = ast, ''
|
||
|
|
|
||
|
|
status = 0
|
||
|
|
for cmd in cmds:
|
||
|
|
try:
|
||
|
|
status = self.execute(cmd)
|
||
|
|
except ExitSignal as e:
|
||
|
|
if sourced:
|
||
|
|
raise
|
||
|
|
status = int(e.args[0])
|
||
|
|
return status
|
||
|
|
except ShellError:
|
||
|
|
self._env['?'] = 1
|
||
|
|
raise
|
||
|
|
if 'debug-utility' in self._debugflags or 'debug-cmd' in self._debugflags:
|
||
|
|
self.log('returncode ' + str(status)+ '\n')
|
||
|
|
return status
|
||
|
|
except CommandNotFound as e:
|
||
|
|
print >>self._redirs.stderr, str(e)
|
||
|
|
self._redirs.stderr.flush()
|
||
|
|
# Command not found by non-interactive shell
|
||
|
|
# return 127
|
||
|
|
raise
|
||
|
|
except RedirectionError as e:
|
||
|
|
# TODO: should be handled depending on the utility status
|
||
|
|
print >>self._redirs.stderr, str(e)
|
||
|
|
self._redirs.stderr.flush()
|
||
|
|
# Command not found by non-interactive shell
|
||
|
|
# return 127
|
||
|
|
raise
|
||
|
|
|
||
|
|
def dotcommand(self, env, args):
|
||
|
|
if len(args) < 1:
|
||
|
|
raise ShellError('. expects at least one argument')
|
||
|
|
path = args[0]
|
||
|
|
if '/' not in path:
|
||
|
|
found = env.find_in_path(args[0], True)
|
||
|
|
if found:
|
||
|
|
path = found[0]
|
||
|
|
script = file(path).read()
|
||
|
|
return self.execute_script(script=script, sourced=True)
|
||
|
|
|
||
|
|
def execute(self, token, redirs=None):
|
||
|
|
"""Execute and AST subtree with supplied redirections overriding default
|
||
|
|
interpreter ones.
|
||
|
|
Return the exit status.
|
||
|
|
"""
|
||
|
|
if not token:
|
||
|
|
return 0
|
||
|
|
|
||
|
|
if redirs is None:
|
||
|
|
redirs = self._redirs
|
||
|
|
|
||
|
|
if isinstance(token, list):
|
||
|
|
# Commands sequence
|
||
|
|
res = 0
|
||
|
|
for t in token:
|
||
|
|
res = self.execute(t, redirs)
|
||
|
|
return res
|
||
|
|
|
||
|
|
type, value = token
|
||
|
|
status = 0
|
||
|
|
if type=='simple_command':
|
||
|
|
redirs_copy = redirs.clone()
|
||
|
|
try:
|
||
|
|
# TODO: define and handle command return values
|
||
|
|
# TODO: implement set -e
|
||
|
|
status = self._execute_simple_command(value, redirs_copy)
|
||
|
|
finally:
|
||
|
|
redirs_copy.close()
|
||
|
|
elif type=='pipeline':
|
||
|
|
status = self._execute_pipeline(value, redirs)
|
||
|
|
elif type=='and_or':
|
||
|
|
status = self._execute_and_or(value, redirs)
|
||
|
|
elif type=='for_clause':
|
||
|
|
status = self._execute_for_clause(value, redirs)
|
||
|
|
elif type=='while_clause':
|
||
|
|
status = self._execute_while_clause(value, redirs)
|
||
|
|
elif type=='function_definition':
|
||
|
|
status = self._execute_function_definition(value, redirs)
|
||
|
|
elif type=='brace_group':
|
||
|
|
status = self._execute_brace_group(value, redirs)
|
||
|
|
elif type=='if_clause':
|
||
|
|
status = self._execute_if_clause(value, redirs)
|
||
|
|
elif type=='subshell':
|
||
|
|
status = self.subshell(ast=value.cmds, redirs=redirs)
|
||
|
|
elif type=='async':
|
||
|
|
status = self._asynclist(value)
|
||
|
|
elif type=='redirect_list':
|
||
|
|
redirs_copy = self.redirect(redirs.clone(), value.redirs)
|
||
|
|
try:
|
||
|
|
status = self.execute(value.cmd, redirs_copy)
|
||
|
|
finally:
|
||
|
|
redirs_copy.close()
|
||
|
|
else:
|
||
|
|
raise NotImplementedError('Unsupported token type ' + type)
|
||
|
|
|
||
|
|
if status < 0:
|
||
|
|
status = 255
|
||
|
|
return status
|
||
|
|
|
||
|
|
def _execute_if_clause(self, if_clause, redirs):
|
||
|
|
cond_status = self.execute(if_clause.cond, redirs)
|
||
|
|
if cond_status==0:
|
||
|
|
return self.execute(if_clause.if_cmds, redirs)
|
||
|
|
else:
|
||
|
|
return self.execute(if_clause.else_cmds, redirs)
|
||
|
|
|
||
|
|
def _execute_brace_group(self, group, redirs):
|
||
|
|
status = 0
|
||
|
|
for cmd in group.cmds:
|
||
|
|
status = self.execute(cmd, redirs)
|
||
|
|
return status
|
||
|
|
|
||
|
|
def _execute_function_definition(self, fundef, redirs):
|
||
|
|
self._env.define_function(fundef.name, fundef.body)
|
||
|
|
return 0
|
||
|
|
|
||
|
|
def _execute_while_clause(self, while_clause, redirs):
|
||
|
|
status = 0
|
||
|
|
while 1:
|
||
|
|
cond_status = 0
|
||
|
|
for cond in while_clause.condition:
|
||
|
|
cond_status = self.execute(cond, redirs)
|
||
|
|
|
||
|
|
if cond_status:
|
||
|
|
break
|
||
|
|
|
||
|
|
for cmd in while_clause.cmds:
|
||
|
|
status = self.execute(cmd, redirs)
|
||
|
|
|
||
|
|
return status
|
||
|
|
|
||
|
|
def _execute_for_clause(self, for_clause, redirs):
|
||
|
|
if not is_name(for_clause.name):
|
||
|
|
raise ShellSyntaxError('%s is not a valid name' % repr(for_clause.name))
|
||
|
|
items = mappend(self.expand_token, for_clause.items)
|
||
|
|
|
||
|
|
status = 0
|
||
|
|
for item in items:
|
||
|
|
self._env[for_clause.name] = item
|
||
|
|
for cmd in for_clause.cmds:
|
||
|
|
status = self.execute(cmd, redirs)
|
||
|
|
return status
|
||
|
|
|
||
|
|
def _execute_and_or(self, or_and, redirs):
|
||
|
|
res = self.execute(or_and.left, redirs)
|
||
|
|
if (or_and.op=='&&' and res==0) or (or_and.op!='&&' and res!=0):
|
||
|
|
res = self.execute(or_and.right, redirs)
|
||
|
|
return res
|
||
|
|
|
||
|
|
def _execute_pipeline(self, pipeline, redirs):
|
||
|
|
if len(pipeline.commands)==1:
|
||
|
|
status = self.execute(pipeline.commands[0], redirs)
|
||
|
|
else:
|
||
|
|
# Execute all commands one after the other
|
||
|
|
status = 0
|
||
|
|
inpath, outpath = None, None
|
||
|
|
try:
|
||
|
|
# Commands inputs and outputs cannot really be plugged as done
|
||
|
|
# by a real shell. Run commands sequentially and chain their
|
||
|
|
# input/output throught temporary files.
|
||
|
|
tmpfd, inpath = tempfile.mkstemp()
|
||
|
|
os.close(tmpfd)
|
||
|
|
tmpfd, outpath = tempfile.mkstemp()
|
||
|
|
os.close(tmpfd)
|
||
|
|
|
||
|
|
inpath = win32_to_unix_path(inpath)
|
||
|
|
outpath = win32_to_unix_path(outpath)
|
||
|
|
|
||
|
|
for i, cmd in enumerate(pipeline.commands):
|
||
|
|
call_redirs = redirs.clone()
|
||
|
|
try:
|
||
|
|
if i!=0:
|
||
|
|
call_redirs.add(self, '<', inpath)
|
||
|
|
if i!=len(pipeline.commands)-1:
|
||
|
|
call_redirs.add(self, '>', outpath)
|
||
|
|
|
||
|
|
status = self.execute(cmd, call_redirs)
|
||
|
|
|
||
|
|
# Chain inputs/outputs
|
||
|
|
inpath, outpath = outpath, inpath
|
||
|
|
finally:
|
||
|
|
call_redirs.close()
|
||
|
|
finally:
|
||
|
|
if inpath: os.remove(inpath)
|
||
|
|
if outpath: os.remove(outpath)
|
||
|
|
|
||
|
|
if pipeline.reverse_status:
|
||
|
|
status = int(not status)
|
||
|
|
self._env['?'] = status
|
||
|
|
return status
|
||
|
|
|
||
|
|
def _execute_function(self, name, args, interp, env, stdin, stdout, stderr, *others):
|
||
|
|
assert interp is self
|
||
|
|
|
||
|
|
func = env.get_function(name)
|
||
|
|
#Set positional parameters
|
||
|
|
prevargs = None
|
||
|
|
try:
|
||
|
|
prevargs = env.set_positional_args(args)
|
||
|
|
try:
|
||
|
|
redirs = Redirections(stdin.dup(), stdout.dup(), stderr.dup())
|
||
|
|
try:
|
||
|
|
status = self.execute(func, redirs)
|
||
|
|
finally:
|
||
|
|
redirs.close()
|
||
|
|
except ReturnSignal as e:
|
||
|
|
status = int(e.args[0])
|
||
|
|
env['?'] = status
|
||
|
|
return status
|
||
|
|
finally:
|
||
|
|
#Reset positional parameters
|
||
|
|
if prevargs is not None:
|
||
|
|
env.set_positional_args(prevargs)
|
||
|
|
|
||
|
|
def _execute_simple_command(self, token, redirs):
|
||
|
|
"""Can raise ReturnSignal when return builtin is called, ExitSignal when
|
||
|
|
exit is called, and other shell exceptions upon builtin failures.
|
||
|
|
"""
|
||
|
|
debug_command = 'debug-cmd' in self._debugflags
|
||
|
|
if debug_command:
|
||
|
|
self.log('word' + repr(token.words) + '\n')
|
||
|
|
self.log('assigns' + repr(token.assigns) + '\n')
|
||
|
|
self.log('redirs' + repr(token.redirs) + '\n')
|
||
|
|
|
||
|
|
is_special = None
|
||
|
|
env = self._env
|
||
|
|
|
||
|
|
try:
|
||
|
|
# Word expansion
|
||
|
|
args = []
|
||
|
|
for word in token.words:
|
||
|
|
args += self.expand_token(word)
|
||
|
|
if is_special is None and args:
|
||
|
|
is_special = env.is_function(args[0]) or \
|
||
|
|
(args[0] in self.COMMANDS and self.COMMANDS[args[0]].is_special)
|
||
|
|
|
||
|
|
if debug_command:
|
||
|
|
self.log('_execute_simple_command' + str(args) + '\n')
|
||
|
|
|
||
|
|
if not args:
|
||
|
|
# Redirections happen is a subshell
|
||
|
|
redirs = redirs.clone()
|
||
|
|
elif not is_special:
|
||
|
|
env = self._env.clone()
|
||
|
|
|
||
|
|
# Redirections
|
||
|
|
self.redirect(redirs, token.redirs)
|
||
|
|
|
||
|
|
# Variables assignments
|
||
|
|
res = 0
|
||
|
|
for type,(k,v) in token.assigns:
|
||
|
|
status, expanded = self.expand_variable((k,v))
|
||
|
|
if status is not None:
|
||
|
|
res = status
|
||
|
|
if args:
|
||
|
|
env.export(k, expanded)
|
||
|
|
else:
|
||
|
|
env[k] = expanded
|
||
|
|
|
||
|
|
if args and args[0] in ('.', 'source'):
|
||
|
|
res = self.dotcommand(env, args[1:])
|
||
|
|
elif args:
|
||
|
|
if args[0] in self.COMMANDS:
|
||
|
|
command = self.COMMANDS[args[0]]
|
||
|
|
elif env.is_function(args[0]):
|
||
|
|
command = Utility(self._execute_function, is_special=True)
|
||
|
|
else:
|
||
|
|
if not '/' in args[0].replace('\\', '/'):
|
||
|
|
cmd = env.find_in_path(args[0])
|
||
|
|
if not cmd:
|
||
|
|
# TODO: test error code on unknown command => 127
|
||
|
|
raise CommandNotFound('Unknown command: "%s"' % args[0])
|
||
|
|
else:
|
||
|
|
# Handle commands like '/cygdrive/c/foo.bat'
|
||
|
|
cmd = cygwin_to_windows_path(args[0])
|
||
|
|
if not os.path.exists(cmd):
|
||
|
|
raise CommandNotFound('%s: No such file or directory' % args[0])
|
||
|
|
shebang = resolve_shebang(cmd)
|
||
|
|
if shebang:
|
||
|
|
cmd = shebang
|
||
|
|
else:
|
||
|
|
cmd = [cmd]
|
||
|
|
args[0:1] = cmd
|
||
|
|
command = Utility(builtin.run_command)
|
||
|
|
|
||
|
|
# Command execution
|
||
|
|
if 'debug-cmd' in self._debugflags:
|
||
|
|
self.log('redirections ' + str(redirs) + '\n')
|
||
|
|
|
||
|
|
res = command.func(args[0], args[1:], self, env,
|
||
|
|
redirs.stdin(), redirs.stdout(),
|
||
|
|
redirs.stderr(), self._debugflags)
|
||
|
|
|
||
|
|
if self._env.has_opt('-x'):
|
||
|
|
# Trace command execution in shell environment
|
||
|
|
# BUG: would be hard to reproduce a real shell behaviour since
|
||
|
|
# the AST is not annotated with source lines/tokens.
|
||
|
|
self._redirs.stdout().write(' '.join(args))
|
||
|
|
|
||
|
|
except ReturnSignal:
|
||
|
|
raise
|
||
|
|
except ShellError as e:
|
||
|
|
if is_special or isinstance(e, (ExitSignal,
|
||
|
|
ShellSyntaxError, ExpansionError)):
|
||
|
|
raise e
|
||
|
|
self._redirs.stderr().write(str(e)+'\n')
|
||
|
|
return 1
|
||
|
|
|
||
|
|
return res
|
||
|
|
|
||
|
|
def expand_token(self, word):
|
||
|
|
"""Expand a word as specified in [2.6 Word Expansions]. Return the list
|
||
|
|
of expanded words.
|
||
|
|
"""
|
||
|
|
status, wtrees = self._expand_word(word)
|
||
|
|
return map(pyshlex.wordtree_as_string, wtrees)
|
||
|
|
|
||
|
|
def expand_variable(self, word):
|
||
|
|
"""Return a status code (or None if no command expansion occurred)
|
||
|
|
and a single word.
|
||
|
|
"""
|
||
|
|
status, wtrees = self._expand_word(word, pathname=False, split=False)
|
||
|
|
words = map(pyshlex.wordtree_as_string, wtrees)
|
||
|
|
assert len(words)==1
|
||
|
|
return status, words[0]
|
||
|
|
|
||
|
|
def expand_here_document(self, word):
|
||
|
|
"""Return the expanded document as a single word. The here document is
|
||
|
|
assumed to be unquoted.
|
||
|
|
"""
|
||
|
|
status, wtrees = self._expand_word(word, pathname=False,
|
||
|
|
split=False, here_document=True)
|
||
|
|
words = map(pyshlex.wordtree_as_string, wtrees)
|
||
|
|
assert len(words)==1
|
||
|
|
return words[0]
|
||
|
|
|
||
|
|
def expand_redirection(self, word):
|
||
|
|
"""Return a single word."""
|
||
|
|
return self.expand_variable(word)[1]
|
||
|
|
|
||
|
|
def get_env(self):
|
||
|
|
return self._env
|
||
|
|
|
||
|
|
def _expand_word(self, token, pathname=True, split=True, here_document=False):
|
||
|
|
wtree = pyshlex.make_wordtree(token[1], here_document=here_document)
|
||
|
|
|
||
|
|
# TODO: implement tilde expansion
|
||
|
|
def expand(wtree):
|
||
|
|
"""Return a pseudo wordtree: the tree or its subelements can be empty
|
||
|
|
lists when no value result from the expansion.
|
||
|
|
"""
|
||
|
|
status = None
|
||
|
|
for part in wtree:
|
||
|
|
if not isinstance(part, list):
|
||
|
|
continue
|
||
|
|
if part[0]in ("'", '\\'):
|
||
|
|
continue
|
||
|
|
elif part[0] in ('`', '$('):
|
||
|
|
status, result = self._expand_command(part)
|
||
|
|
part[:] = result
|
||
|
|
elif part[0] in ('$', '${'):
|
||
|
|
part[:] = self._expand_parameter(part, wtree[0]=='"', split)
|
||
|
|
elif part[0] in ('', '"'):
|
||
|
|
status, result = expand(part)
|
||
|
|
part[:] = result
|
||
|
|
else:
|
||
|
|
raise NotImplementedError('%s expansion is not implemented'
|
||
|
|
% part[0])
|
||
|
|
# [] is returned when an expansion result in no-field,
|
||
|
|
# like an empty $@
|
||
|
|
wtree = [p for p in wtree if p != []]
|
||
|
|
if len(wtree) < 3:
|
||
|
|
return status, []
|
||
|
|
return status, wtree
|
||
|
|
|
||
|
|
status, wtree = expand(wtree)
|
||
|
|
if len(wtree) == 0:
|
||
|
|
return status, wtree
|
||
|
|
wtree = pyshlex.normalize_wordtree(wtree)
|
||
|
|
|
||
|
|
if split:
|
||
|
|
wtrees = self._split_fields(wtree)
|
||
|
|
else:
|
||
|
|
wtrees = [wtree]
|
||
|
|
|
||
|
|
if pathname:
|
||
|
|
wtrees = mappend(self._expand_pathname, wtrees)
|
||
|
|
|
||
|
|
wtrees = map(self._remove_quotes, wtrees)
|
||
|
|
return status, wtrees
|
||
|
|
|
||
|
|
def _expand_command(self, wtree):
|
||
|
|
# BUG: there is something to do with backslashes and quoted
|
||
|
|
# characters here
|
||
|
|
command = pyshlex.wordtree_as_string(wtree[1:-1])
|
||
|
|
status, output = self.subshell_output(command)
|
||
|
|
return status, ['', output, '']
|
||
|
|
|
||
|
|
def _expand_parameter(self, wtree, quoted=False, split=False):
|
||
|
|
"""Return a valid wtree or an empty list when no parameter results."""
|
||
|
|
# Get the parameter name
|
||
|
|
# TODO: implement weird expansion rules with ':'
|
||
|
|
name = pyshlex.wordtree_as_string(wtree[1:-1])
|
||
|
|
if not is_name(name) and not is_special_param(name):
|
||
|
|
raise ExpansionError('Bad substitution "%s"' % name)
|
||
|
|
# TODO: implement special parameters
|
||
|
|
if name in ('@', '*'):
|
||
|
|
args = self._env.get_positional_args()
|
||
|
|
if len(args) == 0:
|
||
|
|
return []
|
||
|
|
if len(args)<2:
|
||
|
|
return ['', ''.join(args), '']
|
||
|
|
|
||
|
|
sep = self._env.get('IFS', '')[:1]
|
||
|
|
if split and quoted and name=='@':
|
||
|
|
# Introduce a new token to tell the caller that these parameters
|
||
|
|
# cause a split as specified in 2.5.2
|
||
|
|
return ['@'] + args + ['']
|
||
|
|
else:
|
||
|
|
return ['', sep.join(args), '']
|
||
|
|
|
||
|
|
return ['', self._env.get(name, ''), '']
|
||
|
|
|
||
|
|
def _split_fields(self, wtree):
|
||
|
|
def is_empty(split):
|
||
|
|
return split==['', '', '']
|
||
|
|
|
||
|
|
def split_positional(quoted):
|
||
|
|
# Return a list of wtree split according positional parameters rules.
|
||
|
|
# All remaining '@' groups are removed.
|
||
|
|
assert quoted[0]=='"'
|
||
|
|
|
||
|
|
splits = [[]]
|
||
|
|
for part in quoted:
|
||
|
|
if not isinstance(part, list) or part[0]!='@':
|
||
|
|
splits[-1].append(part)
|
||
|
|
else:
|
||
|
|
# Empty or single argument list were dealt with already
|
||
|
|
assert len(part)>3
|
||
|
|
# First argument must join with the beginning part of the original word
|
||
|
|
splits[-1].append(part[1])
|
||
|
|
# Create double-quotes expressions for every argument after the first
|
||
|
|
for arg in part[2:-1]:
|
||
|
|
splits[-1].append('"')
|
||
|
|
splits.append(['"', arg])
|
||
|
|
return splits
|
||
|
|
|
||
|
|
# At this point, all expansions but pathnames have occured. Only quoted
|
||
|
|
# and positional sequences remain. Thus, all candidates for field splitting
|
||
|
|
# are in the tree root, or are positional splits ('@') and lie in root
|
||
|
|
# children.
|
||
|
|
if not wtree or wtree[0] not in ('', '"'):
|
||
|
|
# The whole token is quoted or empty, nothing to split
|
||
|
|
return [wtree]
|
||
|
|
|
||
|
|
if wtree[0]=='"':
|
||
|
|
wtree = ['', wtree, '']
|
||
|
|
|
||
|
|
result = [['', '']]
|
||
|
|
for part in wtree[1:-1]:
|
||
|
|
if isinstance(part, list):
|
||
|
|
if part[0]=='"':
|
||
|
|
splits = split_positional(part)
|
||
|
|
if len(splits)<=1:
|
||
|
|
result[-1] += [part, '']
|
||
|
|
else:
|
||
|
|
# Terminate the current split
|
||
|
|
result[-1] += [splits[0], '']
|
||
|
|
result += splits[1:-1]
|
||
|
|
# Create a new split
|
||
|
|
result += [['', splits[-1], '']]
|
||
|
|
else:
|
||
|
|
result[-1] += [part, '']
|
||
|
|
else:
|
||
|
|
splits = self._env.split_fields(part)
|
||
|
|
if len(splits)<=1:
|
||
|
|
# No split
|
||
|
|
result[-1][-1] += part
|
||
|
|
else:
|
||
|
|
# Terminate the current resulting part and create a new one
|
||
|
|
result[-1][-1] += splits[0]
|
||
|
|
result[-1].append('')
|
||
|
|
result += [['', r, ''] for r in splits[1:-1]]
|
||
|
|
result += [['', splits[-1]]]
|
||
|
|
result[-1].append('')
|
||
|
|
|
||
|
|
# Leading and trailing empty groups come from leading/trailing blanks
|
||
|
|
if result and is_empty(result[-1]):
|
||
|
|
result[-1:] = []
|
||
|
|
if result and is_empty(result[0]):
|
||
|
|
result[:1] = []
|
||
|
|
return result
|
||
|
|
|
||
|
|
def _expand_pathname(self, wtree):
|
||
|
|
"""See [2.6.6 Pathname Expansion]."""
|
||
|
|
if self._env.has_opt('-f'):
|
||
|
|
return [wtree]
|
||
|
|
|
||
|
|
# All expansions have been performed, only quoted sequences should remain
|
||
|
|
# in the tree. Generate the pattern by folding the tree, escaping special
|
||
|
|
# characters when appear quoted
|
||
|
|
special_chars = '*?[]'
|
||
|
|
|
||
|
|
def make_pattern(wtree):
|
||
|
|
subpattern = []
|
||
|
|
for part in wtree[1:-1]:
|
||
|
|
if isinstance(part, list):
|
||
|
|
part = make_pattern(part)
|
||
|
|
elif wtree[0]!='':
|
||
|
|
for c in part:
|
||
|
|
# Meta-characters cannot be quoted
|
||
|
|
if c in special_chars:
|
||
|
|
raise GlobError()
|
||
|
|
subpattern.append(part)
|
||
|
|
return ''.join(subpattern)
|
||
|
|
|
||
|
|
def pwd_glob(pattern):
|
||
|
|
cwd = os.getcwd()
|
||
|
|
os.chdir(self._env['PWD'])
|
||
|
|
try:
|
||
|
|
return glob.glob(pattern)
|
||
|
|
finally:
|
||
|
|
os.chdir(cwd)
|
||
|
|
|
||
|
|
#TODO: check working directory issues here wrt relative patterns
|
||
|
|
try:
|
||
|
|
pattern = make_pattern(wtree)
|
||
|
|
paths = pwd_glob(pattern)
|
||
|
|
except GlobError:
|
||
|
|
# BUG: Meta-characters were found in quoted sequences. The should
|
||
|
|
# have been used literally but this is unsupported in current glob module.
|
||
|
|
# Instead we consider the whole tree must be used literally and
|
||
|
|
# therefore there is no point in globbing. This is wrong when meta
|
||
|
|
# characters are mixed with quoted meta in the same pattern like:
|
||
|
|
# < foo*"py*" >
|
||
|
|
paths = []
|
||
|
|
|
||
|
|
if not paths:
|
||
|
|
return [wtree]
|
||
|
|
return [['', path, ''] for path in paths]
|
||
|
|
|
||
|
|
def _remove_quotes(self, wtree):
|
||
|
|
"""See [2.6.7 Quote Removal]."""
|
||
|
|
|
||
|
|
def unquote(wtree):
|
||
|
|
unquoted = []
|
||
|
|
for part in wtree[1:-1]:
|
||
|
|
if isinstance(part, list):
|
||
|
|
part = unquote(part)
|
||
|
|
unquoted.append(part)
|
||
|
|
return ''.join(unquoted)
|
||
|
|
|
||
|
|
return ['', unquote(wtree), '']
|
||
|
|
|
||
|
|
def subshell(self, script=None, ast=None, redirs=None):
|
||
|
|
"""Execute the script or AST in a subshell, with inherited redirections
|
||
|
|
if redirs is not None.
|
||
|
|
"""
|
||
|
|
if redirs:
|
||
|
|
sub_redirs = redirs
|
||
|
|
else:
|
||
|
|
sub_redirs = redirs.clone()
|
||
|
|
|
||
|
|
subshell = None
|
||
|
|
try:
|
||
|
|
subshell = Interpreter(None, self._debugflags, self._env.clone(True),
|
||
|
|
sub_redirs, opts=self._options)
|
||
|
|
return subshell.execute_script(script, ast)
|
||
|
|
finally:
|
||
|
|
if not redirs: sub_redirs.close()
|
||
|
|
if subshell: subshell.close()
|
||
|
|
|
||
|
|
def subshell_output(self, script):
|
||
|
|
"""Execute the script in a subshell and return the captured output."""
|
||
|
|
# Create temporary file to capture subshell output
|
||
|
|
tmpfd, tmppath = tempfile.mkstemp()
|
||
|
|
try:
|
||
|
|
tmpfile = os.fdopen(tmpfd, 'wb')
|
||
|
|
stdout = FileWrapper('w', tmpfile)
|
||
|
|
|
||
|
|
redirs = Redirections(self._redirs.stdin().dup(),
|
||
|
|
stdout,
|
||
|
|
self._redirs.stderr().dup())
|
||
|
|
try:
|
||
|
|
status = self.subshell(script=script, redirs=redirs)
|
||
|
|
finally:
|
||
|
|
redirs.close()
|
||
|
|
redirs = None
|
||
|
|
|
||
|
|
# Extract subshell standard output
|
||
|
|
tmpfile = open(tmppath, 'rb')
|
||
|
|
try:
|
||
|
|
output = tmpfile.read()
|
||
|
|
return status, output.rstrip('\n')
|
||
|
|
finally:
|
||
|
|
tmpfile.close()
|
||
|
|
finally:
|
||
|
|
os.remove(tmppath)
|
||
|
|
|
||
|
|
def _asynclist(self, cmd):
|
||
|
|
args = (self._env.get_variables(), cmd)
|
||
|
|
arg = encodeargs(args)
|
||
|
|
assert len(args) < 30*1024
|
||
|
|
cmd = ['pysh.bat', '--ast', '-c', arg]
|
||
|
|
p = subprocess.Popen(cmd, cwd=self._env['PWD'])
|
||
|
|
self._children[p.pid] = p
|
||
|
|
self._env['!'] = p.pid
|
||
|
|
return 0
|
||
|
|
|
||
|
|
def wait(self, pids=None):
|
||
|
|
if not pids:
|
||
|
|
pids = self._children.keys()
|
||
|
|
|
||
|
|
status = 127
|
||
|
|
for pid in pids:
|
||
|
|
if pid not in self._children:
|
||
|
|
continue
|
||
|
|
p = self._children.pop(pid)
|
||
|
|
status = p.wait()
|
||
|
|
|
||
|
|
return status
|
||
|
|
|