From 46a206ae3046ee02f03eb07e28c980d0b277c833 Mon Sep 17 00:00:00 2001 From: Benedek Racz Date: Tue, 21 Jan 2020 09:36:46 +0100 Subject: [PATCH] [ADD] SpawnPipe, and SpawnSocket classes --- tests/test_command_list_split.py | 8 +- wexpect/__init__.py | 7 +- wexpect/console_reader.py | 76 +++---- wexpect/spawn.py | 326 ++++++++++++++++++------------- wexpect/wexpect_util.py | 65 ++++-- 5 files changed, 290 insertions(+), 192 deletions(-) diff --git a/tests/test_command_list_split.py b/tests/test_command_list_split.py index 9ae5396..de8b9b2 100644 --- a/tests/test_command_list_split.py +++ b/tests/test_command_list_split.py @@ -22,8 +22,8 @@ import wexpect import unittest from . import PexpectTestCase -class SplitCommandLineTestCase(PexpectTestCase.PexpectTestCase): - def testSplitSizes(self): +class TestCaseSplitCommandLine(PexpectTestCase.PexpectTestCase): + def test_split_sizes(self): self.assertEqual(len(wexpect.split_command_line(r'')), 0) self.assertEqual(len(wexpect.split_command_line(r'one')), 1) self.assertEqual(len(wexpect.split_command_line(r'one two')), 2) @@ -34,7 +34,7 @@ class SplitCommandLineTestCase(PexpectTestCase.PexpectTestCase): self.assertEqual(len(wexpect.split_command_line(r'one\"one')), 1) self.assertEqual(len(wexpect.split_command_line(r"This^' is a^'^ test")), 3) - def testJoinArgs(self): + def test_join_args(self): cmd = 'foo bar "b a z"' cmd2 = wexpect.join_args(wexpect.split_command_line(cmd)) self.assertEqual(cmd2, cmd) @@ -46,4 +46,4 @@ class SplitCommandLineTestCase(PexpectTestCase.PexpectTestCase): if __name__ == '__main__': unittest.main() -suite = unittest.makeSuite(SplitCommandLineTestCase,'test') +suite = unittest.makeSuite(TestCaseSplitCommandLine,'test') diff --git a/wexpect/__init__.py b/wexpect/__init__.py index f7b46ea..27ffa56 100644 --- a/wexpect/__init__.py +++ b/wexpect/__init__.py @@ -9,9 +9,10 @@ from .wexpect_util import TIMEOUT from .console_reader import ConsoleReaderSocket from .console_reader import ConsoleReaderPipe -from .spawn import Spawn -from .spawn import Spawn as spawn +from .spawn import SpawnSocket +from .spawn import SpawnPipe +from .spawn import SpawnSocket as spawn from .spawn import run __all__ = ['split_command_line', 'join_args', 'ExceptionPexpect', 'EOF', 'TIMEOUT', - 'ConsoleReaderSocket', 'ConsoleReaderPipe', 'spawn', 'Spawn', 'run'] + 'ConsoleReaderSocket', 'ConsoleReaderPipe', 'spawn', 'SpawnSocket', 'SpawnPipe', 'run'] diff --git a/wexpect/console_reader.py b/wexpect/console_reader.py index cf9cb39..e5b2f05 100644 --- a/wexpect/console_reader.py +++ b/wexpect/console_reader.py @@ -72,16 +72,24 @@ except: # pragma: no cover # console manipulation. # logger = logging.getLogger('wexpect') -os.environ['WEXPECT_LOGGER_LEVEL'] = 'DEBUG' -try: - logger_level = os.environ['WEXPECT_LOGGER_LEVEL'] - logger.setLevel(logger_level) - fh = logging.FileHandler('wexpect.log', 'w', 'utf-8') - formatter = logging.Formatter('%(asctime)s - %(filename)s::%(funcName)s - %(levelname)s - %(message)s') - fh.setFormatter(formatter) - logger.addHandler(fh) -except KeyError: - logger.setLevel(logging.ERROR) + +def init_logger(): + logger = logging.getLogger('wexpect') + os.environ['WEXPECT_LOGGER_LEVEL'] = 'DEBUG' + try: + logger_level = os.environ['WEXPECT_LOGGER_LEVEL'] + try: + logger_filename = os.environ['WEXPECT_LOGGER_FILENAME'] + except KeyError: + pid = os.getpid() + logger_filename = f'wexpect_{pid}' + logger.setLevel(logger_level) + fh = logging.FileHandler(f'{logger_filename}.log', 'w', 'utf-8') + formatter = logging.Formatter('%(asctime)s - %(filename)s::%(funcName)s - %(levelname)s - %(message)s') + fh.setFormatter(formatter) + logger.addHandler(fh) + except KeyError: + logger.setLevel(logging.ERROR) class ConsoleReaderBase: """Consol class (aka. client-side python class) for the child. @@ -109,7 +117,9 @@ class ConsoleReaderBase: self.consin = None self.consout = None self.local_echo = True + self.pid = os.getpid() + init_logger() logger.info("ConsoleReader started") if cp: @@ -129,13 +139,8 @@ class ConsoleReaderBase: self.__childProcess, _, childPid, self.__tid = win32process.CreateProcess(None, path, None, None, False, 0, None, None, si) - print('123') - print('456') - print('789') - except Exception as e: logger.info(e) - time.sleep(.1) return time.sleep(.2) @@ -162,10 +167,6 @@ class ConsoleReaderBase: """ if e.args[0] != winerror.ERROR_ACCESS_DENIED: logger.info(e) - - time.sleep(.1) - self.send_to_host(self.readConsoleToCursor()) - time.sleep(.1) return if cursorPos.Y > maxconsoleY and not paused: @@ -183,6 +184,9 @@ class ConsoleReaderBase: logger.error(traceback.format_exc()) time.sleep(.1) finally: + time.sleep(.1) + self.send_to_host(self.readConsoleToCursor()) + time.sleep(1) self.close_connection() def write(self, s): @@ -405,19 +409,22 @@ class ConsoleReaderSocket(ConsoleReaderBase): def create_connection(self, **kwargs): - - self.port = kwargs['port'] - # Create a TCP/IP socket - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_address = ('localhost', self.port) - self.sock.bind(server_address) - logger.info(f'Socket started at port: {self.port}') - - # Listen for incoming connections - self.sock.listen(1) - self.connection, client_address = self.sock.accept() - self.connection.settimeout(.2) - logger.info(f'Client connected: {client_address}') + try: + self.port = kwargs['port'] + # Create a TCP/IP socket + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_address = ('localhost', self.port) + self.sock.bind(server_address) + logger.info(f'Socket started at port: {self.port}') + + # Listen for incoming connections + self.sock.listen(1) + self.connection, client_address = self.sock.accept() + self.connection.settimeout(.2) + logger.info(f'Client connected: {client_address}') + except: + logger.error(f"Port: {self.port}") + raise def close_connection(self): if self.connection: @@ -449,9 +456,8 @@ class ConsoleReaderSocket(ConsoleReaderBase): class ConsoleReaderPipe(ConsoleReaderBase): - def create_connection(self): - pid = win32process.GetCurrentProcessId() - pipe_name = 'wexpect_{}'.format(pid) + def create_connection(self, **kwargs): + pipe_name = 'wexpect_{}'.format(self.pid) pipe_full_path = r'\\.\pipe\{}'.format(pipe_name) logger.info('Start pipe server: %s', pipe_full_path) self.pipe = win32pipe.CreateNamedPipe( diff --git a/wexpect/spawn.py b/wexpect/spawn.py index a1db127..15f62a7 100644 --- a/wexpect/spawn.py +++ b/wexpect/spawn.py @@ -150,9 +150,9 @@ def run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None dictionary passed to a callback. """ if timeout == -1: - child = Spawn(command, maxread=2000, logfile=logfile, cwd=cwd, env=env) + child = SpawnSocket(command, maxread=2000, logfile=logfile, cwd=cwd, env=env) else: - child = Spawn(command, timeout=timeout, maxread=2000, logfile=logfile, cwd=cwd, env=env) + child = SpawnSocket(command, timeout=timeout, maxread=2000, logfile=logfile, cwd=cwd, env=env) if events is not None: patterns = list(events.keys()) responses = list(events.values()) @@ -194,7 +194,7 @@ def run (command, timeout=-1, withexitstatus=False, events=None, extra_args=None return child_result -class Spawn: +class SpawnBase: def __init__(self, command, args=[], timeout=30, maxread=60000, searchwindowsize=None, logfile=None, cwd=None, env=None, codepage=None, echo=True): """This starts the given command in a child process. This does all the @@ -271,7 +271,7 @@ class Spawn: self.closed = False self.child_fd = self.startChild(self.args, self.env) - self.connect_to_child('localhost', 4321) + self.connect_to_child() def __del__(self): """This makes sure that no system resources are left open. Python only @@ -391,36 +391,9 @@ class Spawn: return self def read_nonblocking (self, size = 1): - """This reads at most size characters from the child application. If - the end of file is read then an EOF exception will be raised. - - This is not effected by the 'size' parameter, so if you call - read_nonblocking(size=100, timeout=30) and only one character is - available right away then one character will be returned immediately. - It will not wait for 30 seconds for another 99 characters to come in. - - This is a wrapper around Wtty.read(). """ - - if self.closed: - raise ValueError ('I/O operation on closed file in read_nonblocking().') - - try: - # The real child and it's console are two different process. The console dies 0.1 sec - # later to be able to read the child's last output (before EOF). So here we check - # isalive() (which checks the real child.) and try a last read on the console. To catch - # the last output. - # The flag_child_finished flag shows that this is the second trial, where we raise the EOF. - if self.flag_child_finished: - raise EOF('self.flag_child_finished') - if not self.isalive(): - self.flag_child_finished = True - - s = self.sock.recv(size) - except EOF: - self.flag_eof = True - raise - - return s.decode() + """Virtual definition + """ + raise NotImplementedError def __next__ (self): # File-like object. @@ -455,101 +428,12 @@ class Spawn: """The child is always created with a console.""" return True - - - def pipe_client(self, conpid): - pipe_name = 'wexpect_{}'.format(conpid) - pipe_full_path = r'\\.\pipe\{}'.format(pipe_name) - print('Trying to connect to pipe: {}'.format(pipe_full_path)) - quit = False - while not quit: - try: - handle = win32file.CreateFile( - pipe_full_path, - win32file.GENERIC_READ | win32file.GENERIC_WRITE, - 0, - None, - win32file.OPEN_EXISTING, - 0, - None - ) - print("pipe found!") - res = win32pipe.SetNamedPipeHandleState(handle, win32pipe.PIPE_READMODE_MESSAGE, None, None) - if res == 0: - print(f"SetNamedPipeHandleState return code: {res}") - while True: - resp = win32file.ReadFile(handle, 64*1024) - print(f"message: {resp}") - win32file.WriteFile(handle, b'back') - except pywintypes.error as e: - if e.args[0] == winerror.ERROR_FILE_NOT_FOUND: #2 - print("no pipe, trying again in a bit later") - time.sleep(0.2) - elif e.args[0] == winerror.ERROR_BROKEN_PIPE: #109 - print("broken pipe, bye bye") - quit = True - elif e.args[0] == winerror.ERROR_NO_DATA: - '''232 (0xE8) - The pipe is being closed. - ''' - print("The pipe is being closed.") - quit = True - else: - raise - - - def connect_to_child(self, host, port): - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.sock.connect((host, port)) - - def disconnect_from_child(self): - if self.sock: - self.sock.close() - - - def startChild(self, args, env): - si = win32process.GetStartupInfo() - si.dwFlags = win32process.STARTF_USESHOWWINDOW - si.wShowWindow = win32con.SW_HIDE - - dirname = os.path.dirname(sys.executable - if getattr(sys, 'frozen', False) else - os.path.abspath(__file__)) - spath = [os.path.dirname(dirname)] - pyargs = ['-c'] - if getattr(sys, 'frozen', False): - # If we are running 'frozen', add library.zip and lib\library.zip - # to sys.path - # py2exe: Needs appropriate 'zipfile' option in setup script and - # 'bundle_files' 3 - spath.append(os.path.join(dirname, 'library.zip')) - spath.append(os.path.join(dirname, 'library.zip', - os.path.basename(os.path.splitext(sys.executable)[0]))) - if os.path.isdir(os.path.join(dirname, 'lib')): - dirname = os.path.join(dirname, 'lib') - spath.append(os.path.join(dirname, 'library.zip')) - spath.append(os.path.join(dirname, 'library.zip', - os.path.basename(os.path.splitext(sys.executable)[0]))) - pyargs.insert(0, '-S') # skip 'import site' - - - pid = win32process.GetCurrentProcessId() - - commandLine = '"%s" %s "%s"' % (os.path.join(dirname, 'python.exe') - if getattr(sys, 'frozen', False) else - os.path.join(os.path.dirname(sys.executable), 'python.exe'), - ' '.join(pyargs), - "import sys;" - f"sys.path = {spath} + sys.path;" - "import wexpect;" - "import time;" - f"wexpect.ConsoleReaderSocket(wexpect.join_args({args}), {pid}, port=4321);" - ) - - self.conproc, _, conpid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, - win32process.CREATE_NEW_CONSOLE, None, None, si) + def kill(self, sig): + """Sig == sigint for ctrl-c otherwise the child is terminated.""" + os.kill(self.conpid, sig) +# win32api.TerminateProcess(self.conproc, 1) def isalive(self, console=True): """True if the child is still alive, false otherwise""" @@ -600,16 +484,21 @@ class Spawn: # platform does not define VEOF so assume CTRL-D char = chr(4) self.send(char) - - def send(self, s): - """This sends a string to the child process. This returns the number of - bytes written. If a log file was set then the data is also written to - the log. """ - time.sleep(self.delaybeforesend) - self.sock.sendall(s) - return len(s) - + def send(self): + """Virtual definition + """ + raise NotImplementedError + + def connect_to_child(self): + """Virtual definition + """ + raise NotImplementedError + + def disconnect_from_child(self): + """Virtual definition + """ + raise NotImplementedError def compile_pattern_list(self, patterns): @@ -846,6 +735,167 @@ class Spawn: self.match_index = None raise +class SpawnPipe(SpawnBase): + + + + def pipe_client(self, conpid): + pipe_name = 'wexpect_{}'.format(conpid) + pipe_full_path = r'\\.\pipe\{}'.format(pipe_name) + print('Trying to connect to pipe: {}'.format(pipe_full_path)) + quit = False + + while not quit: + try: + handle = win32file.CreateFile( + pipe_full_path, + win32file.GENERIC_READ | win32file.GENERIC_WRITE, + 0, + None, + win32file.OPEN_EXISTING, + 0, + None + ) + print("pipe found!") + res = win32pipe.SetNamedPipeHandleState(handle, win32pipe.PIPE_READMODE_MESSAGE, None, None) + if res == 0: + print(f"SetNamedPipeHandleState return code: {res}") + while True: + resp = win32file.ReadFile(handle, 64*1024) + print(f"message: {resp}") + win32file.WriteFile(handle, b'back') + except pywintypes.error as e: + if e.args[0] == winerror.ERROR_FILE_NOT_FOUND: #2 + print("no pipe, trying again in a bit later") + time.sleep(0.2) + elif e.args[0] == winerror.ERROR_BROKEN_PIPE: #109 + print("broken pipe, bye bye") + quit = True + elif e.args[0] == winerror.ERROR_NO_DATA: + '''232 (0xE8) + The pipe is being closed. + ''' + print("The pipe is being closed.") + quit = True + else: + raise + + def send(self, s): + """This sends a string to the child process. This returns the number of + bytes written. If a log file was set then the data is also written to + the log. """ + if isinstance(s, str): + s = str.encode(s) + if self.delaybeforesend: + time.sleep(self.delaybeforesend) + self.sock.sendall(s) + return len(s) + + +class SpawnSocket(SpawnBase): + + def __init__(self, command, args=[], timeout=30, maxread=60000, searchwindowsize=None, + logfile=None, cwd=None, env=None, codepage=None, echo=True, port=4321, host='localhost'): + self.port = port + self.host = host + super().__init__(command=command, args=args, timeout=timeout, maxread=maxread, + searchwindowsize=searchwindowsize, cwd=cwd, env=env, codepage=codepage, echo=echo) + + + def send(self, s): + """This sends a string to the child process. This returns the number of + bytes written. If a log file was set then the data is also written to + the log. """ + if isinstance(s, str): + s = str.encode(s) + if self.delaybeforesend: + time.sleep(self.delaybeforesend) + self.sock.sendall(s) + return len(s) + + def connect_to_child(self): + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.connect((self.host, self.port)) + + def disconnect_from_child(self): + if self.sock: + self.sock.close() + + def read_nonblocking (self, size = 1): + """This reads at most size characters from the child application. If + the end of file is read then an EOF exception will be raised. + + This is not effected by the 'size' parameter, so if you call + read_nonblocking(size=100, timeout=30) and only one character is + available right away then one character will be returned immediately. + It will not wait for 30 seconds for another 99 characters to come in. + + This is a wrapper around Wtty.read(). """ + + if self.closed: + raise ValueError ('I/O operation on closed file in read_nonblocking().') + + try: + # The real child and it's console are two different process. The console dies 0.1 sec + # later to be able to read the child's last output (before EOF). So here we check + # isalive() (which checks the real child.) and try a last read on the console. To catch + # the last output. + # The flag_child_finished flag shows that this is the second trial, where we raise the EOF. + if self.flag_child_finished: + raise EOF('self.flag_child_finished') + if not self.isalive(): + self.flag_child_finished = True + + s = self.sock.recv(size) + except EOF: + self.flag_eof = True + raise + + return s.decode() + + def startChild(self, args, env): + si = win32process.GetStartupInfo() + si.dwFlags = win32process.STARTF_USESHOWWINDOW + si.wShowWindow = win32con.SW_HIDE + + dirname = os.path.dirname(sys.executable + if getattr(sys, 'frozen', False) else + os.path.abspath(__file__)) + spath = [os.path.dirname(dirname)] + pyargs = ['-c'] + if getattr(sys, 'frozen', False): + # If we are running 'frozen', add library.zip and lib\library.zip + # to sys.path + # py2exe: Needs appropriate 'zipfile' option in setup script and + # 'bundle_files' 3 + spath.append(os.path.join(dirname, 'library.zip')) + spath.append(os.path.join(dirname, 'library.zip', + os.path.basename(os.path.splitext(sys.executable)[0]))) + if os.path.isdir(os.path.join(dirname, 'lib')): + dirname = os.path.join(dirname, 'lib') + spath.append(os.path.join(dirname, 'library.zip')) + spath.append(os.path.join(dirname, 'library.zip', + os.path.basename(os.path.splitext(sys.executable)[0]))) + pyargs.insert(0, '-S') # skip 'import site' + + + pid = win32process.GetCurrentProcessId() + + commandLine = '"%s" %s "%s"' % (os.path.join(dirname, 'python.exe') + if getattr(sys, 'frozen', False) else + os.path.join(os.path.dirname(sys.executable), 'python.exe'), + ' '.join(pyargs), + "import sys;" + f"sys.path = {spath} + sys.path;" + "import wexpect;" + "import time;" + f"wexpect.ConsoleReaderSocket(wexpect.join_args({args}), {pid}, port={self.port});" + ) + + self.conproc, _, self.conpid, __otid = win32process.CreateProcess(None, commandLine, None, None, False, + win32process.CREATE_NEW_CONSOLE, None, None, si) + + class searcher_re (object): @@ -938,7 +988,7 @@ class searcher_re (object): def main(): try: - p = Spawn('cmd') + p = SpawnSocket('cmd') p.sendline(b'ls') time.sleep(.5) @@ -957,4 +1007,4 @@ def main(): if __name__ == '__main__': main() - \ No newline at end of file + diff --git a/wexpect/wexpect_util.py b/wexpect/wexpect_util.py index 5c0e93e..ee17e8c 100644 --- a/wexpect/wexpect_util.py +++ b/wexpect/wexpect_util.py @@ -38,21 +38,62 @@ Wexpect Copyright (c) 2019 Benedek Racz """ import re -import ctypes import traceback import sys -def split_command_line(command_line): - '''https://stackoverflow.com/a/35900070/2506522 - ''' - - nargs = ctypes.c_int() - ctypes.windll.shell32.CommandLineToArgvW.restype = ctypes.POINTER(ctypes.c_wchar_p) - lpargs = ctypes.windll.shell32.CommandLineToArgvW(command_line, ctypes.byref(nargs)) - args = [lpargs[i] for i in range(nargs.value)] - if ctypes.windll.kernel32.LocalFree(lpargs): - raise AssertionError - return args +def split_command_line(command_line, escape_char = '^'): + """This splits a command line into a list of arguments. It splits arguments + on spaces, but handles embedded quotes, doublequotes, and escaped + characters. It's impossible to do this with a regular expression, so I + wrote a little state machine to parse the command line. """ + + arg_list = [] + arg = '' + + # Constants to name the states we can be in. + state_basic = 0 + state_esc = 1 + state_singlequote = 2 + state_doublequote = 3 + state_whitespace = 4 # The state of consuming whitespace between commands. + state = state_basic + + for c in command_line: + if state == state_basic or state == state_whitespace: + if c == escape_char: # Escape the next character + state = state_esc + elif c == r"'": # Handle single quote + state = state_singlequote + elif c == r'"': # Handle double quote + state = state_doublequote + elif c.isspace(): + # Add arg to arg_list if we aren't in the middle of whitespace. + if state == state_whitespace: + None # Do nothing. + else: + arg_list.append(arg) + arg = '' + state = state_whitespace + else: + arg = arg + c + state = state_basic + elif state == state_esc: + arg = arg + c + state = state_basic + elif state == state_singlequote: + if c == r"'": + state = state_basic + else: + arg = arg + c + elif state == state_doublequote: + if c == r'"': + state = state_basic + else: + arg = arg + c + + if arg != '': + arg_list.append(arg) + return arg_list def join_args(args): """Joins arguments into a command line. It quotes all arguments that contain